aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichał Górny <mgorny@gentoo.org>2019-10-06 21:44:55 +0200
committerMichał Górny <mgorny@gentoo.org>2019-10-09 08:00:35 +0200
commit0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c (patch)
tree0b314b3bb24c0d38c8177caf4d34598adbbe3cea /lib/portage/_emirrordist/FetchTask.py
parentfetch: Support GLEP 75 mirror structure (diff)
downloadportage-0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c.tar.gz
portage-0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c.tar.bz2
portage-0d34d89d5028ac24cb45ed43ee1eed9cc9836e0c.zip
emirrordist: Implement mirror layout.conf support
Read mirror layout from layout.conf in distfiles directory. When multiple layouts are used, fetch according to the first one and then hardlink according to the remaining layouts. Similarly, when deleting recycle or delete according to the first layout, then unlink according to all remaining layouts. Reviewed-by: Zac Medico <zmedico@gentoo.org> Closes: https://github.com/gentoo/portage/pull/463 Signed-off-by: Michał Górny <mgorny@gentoo.org>
Diffstat (limited to 'lib/portage/_emirrordist/FetchTask.py')
-rw-r--r--lib/portage/_emirrordist/FetchTask.py60
1 files changed, 48 insertions, 12 deletions
diff --git a/lib/portage/_emirrordist/FetchTask.py b/lib/portage/_emirrordist/FetchTask.py
index 1440b697c..7b68b7d3e 100644
--- a/lib/portage/_emirrordist/FetchTask.py
+++ b/lib/portage/_emirrordist/FetchTask.py
@@ -1,4 +1,4 @@
-# Copyright 2013-2014 Gentoo Foundation
+# Copyright 2013-2019 Gentoo Authors
# Distributed under the terms of the GNU General Public License v2
from __future__ import division
@@ -14,6 +14,7 @@ import sys
import portage
from portage import _encodings, _unicode_encode
from portage import os
+from portage.util import ensure_dirs
from portage.util._async.FileCopier import FileCopier
from portage.util._async.FileDigester import FileDigester
from portage.util._async.PipeLogger import PipeLogger
@@ -61,7 +62,8 @@ class FetchTask(CompositeTask):
return
distfile_path = os.path.join(
- self.config.options.distfiles, self.distfile)
+ self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
st = None
size_ok = False
@@ -332,7 +334,8 @@ class FetchTask(CompositeTask):
return
else:
src = os.path.join(current_mirror.location, self.distfile)
- dest = os.path.join(self.config.options.distfiles, self.distfile)
+ dest = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
if self._hardlink_atomic(src, dest,
"%s to %s" % (current_mirror.name, "distfiles")):
logging.debug("hardlink '%s' from %s to distfiles" %
@@ -501,7 +504,9 @@ class FetchTask(CompositeTask):
except OSError:
pass
else:
- dest = os.path.join(self.config.options.distfiles, self.distfile)
+ dest = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
+ ensure_dirs(os.path.dirname(dest))
try:
os.rename(self._fetch_tmp_file, dest)
except OSError:
@@ -514,9 +519,7 @@ class FetchTask(CompositeTask):
self._fetch_copier_exit)
return
else:
- self._success()
- self.returncode = os.EX_OK
- self.wait()
+ self._make_layout_links()
return
self._try_next_mirror()
@@ -535,9 +538,7 @@ class FetchTask(CompositeTask):
return
if copier.returncode == os.EX_OK:
- self._success()
- self.returncode = os.EX_OK
- self.wait()
+ self._make_layout_links()
else:
# out of space?
msg = "%s %s copy failed unexpectedly" % \
@@ -551,6 +552,38 @@ class FetchTask(CompositeTask):
self.returncode = 1
self.wait()
+ def _make_layout_links(self):
+ dist_path = None
+ success = True
+ for layout in self.config.layouts[1:]:
+ if dist_path is None:
+ dist_path = os.path.join(self.config.options.distfiles,
+ self.config.layouts[0].get_path(self.distfile))
+ link_path = os.path.join(self.config.options.distfiles,
+ layout.get_path(self.distfile))
+ ensure_dirs(os.path.dirname(link_path))
+ src_path = dist_path
+ if self.config.options.symlinks:
+ src_path = os.path.relpath(dist_path,
+ os.path.dirname(link_path))
+
+ if not self._hardlink_atomic(src_path, link_path,
+ "%s -> %s" % (link_path, src_path),
+ self.config.options.symlinks):
+ success = False
+ break
+
+ if success:
+ self._success()
+ self.returncode = os.EX_OK
+ else:
+ self.config.log_failure("%s\t%s\t%s" %
+ (self.cpv, self.distfile, msg))
+ self.config.file_failures[self.distfile] = self.cpv
+ self.returncode = 1
+
+ self.wait()
+
def _unlink_file(self, file_path, dir_info):
try:
os.unlink(file_path)
@@ -595,7 +628,7 @@ class FetchTask(CompositeTask):
else:
return st1.st_dev == st2.st_dev
- def _hardlink_atomic(self, src, dest, dir_info):
+ def _hardlink_atomic(self, src, dest, dir_info, symlink=False):
head, tail = os.path.split(dest)
hardlink_tmp = os.path.join(head, ".%s._mirrordist_hardlink_.%s" % \
@@ -603,7 +636,10 @@ class FetchTask(CompositeTask):
try:
try:
- os.link(src, hardlink_tmp)
+ if symlink:
+ os.symlink(src, hardlink_tmp)
+ else:
+ os.link(src, hardlink_tmp)
except OSError as e:
if e.errno != errno.EXDEV:
msg = "hardlink %s from %s failed: %s" % \