Merge branch 'master' of git+ssh://git.gentoo.org/proj/portage

11 files changed, 139 insertions, 58 deletions

diff --git a/bin/egencache b/bin/egencache
index 51d115ade..a9d4657b9 100755
--- a/bin/egencache
+++ b/bin/egencache
@@ -55,7 +55,9 @@ from portage.const import TIMESTAMP_FORMAT
 from portage.manifest import guessManifestFileType
 from portage.package.ebuild._parallel_manifest.ManifestScheduler import ManifestScheduler
 from portage.util import cmp_sort_key, writemsg_level
+from portage.util._async.AsyncFunction import AsyncFunction
 from portage.util._async.run_main_scheduler import run_main_scheduler
+from portage.util._async.TaskScheduler import TaskScheduler
 from portage.util._eventloop.global_event_loop import global_event_loop
 from portage import cpv_getkey
 from portage.dep import Atom, isjustname
@@ -748,7 +750,8 @@ class _special_filename(_filename_base):
 			return self.file_name < other.file_name
 
 class GenChangeLogs(object):
-	def __init__(self, portdb, changelog_output, changelog_reversed):
+	def __init__(self, portdb, changelog_output, changelog_reversed,
+		max_jobs=None, max_load=None):
 		self.returncode = os.EX_OK
 		self._portdb = portdb
 		self._wrapper = textwrap.TextWrapper(
@@ -758,6 +761,8 @@ class GenChangeLogs(object):
 			)
 		self._changelog_output = changelog_output
 		self._changelog_reversed = changelog_reversed
+		self._max_jobs = max_jobs
+		self._max_load = max_load
 
 	@staticmethod
 	def grab(cmd):
@@ -882,7 +887,7 @@ class GenChangeLogs(object):
 
 		output.close()
 
-	def run(self):
+	def _task_iter(self):
 		repo_path = self._portdb.porttrees[0]
 		os.chdir(repo_path)
 
@@ -908,7 +913,12 @@ class GenChangeLogs(object):
 				cmod = 0
 
 			if float(cmod) < float(lmod):
-				self.generate_changelog(cp)
+				yield AsyncFunction(target=self.generate_changelog, args=[cp])
+
+	def run(self):
+		return run_main_scheduler(
+			TaskScheduler(self._task_iter(), event_loop=global_event_loop(),
+				max_jobs=self._max_jobs, max_load=self._max_load))
 
 def egencache_main(args):
 
@@ -1095,29 +1105,6 @@ def egencache_main(args):
 		else:
 			ret.append(gen_cache.returncode)
 
-	if options.update_manifests:
-
-		cp_iter = None
-		if atoms:
-			cp_iter = iter(atoms)
-
-		event_loop = global_event_loop()
-		scheduler = ManifestScheduler(portdb, cp_iter=cp_iter,
-			gpg_cmd=gpg_cmd, gpg_vars=gpg_vars,
-			force_sign_key=force_sign_key,
-			max_jobs=options.jobs,
-			max_load=options.load_average,
-			event_loop=event_loop)
-
-		signum = run_main_scheduler(scheduler)
-		if signum is not None:
-			sys.exit(128 + signum)
-
-		if options.tolerant:
-			ret.append(os.EX_OK)
-		else:
-			ret.append(scheduler.returncode)
-
 	if options.update_pkg_desc_index:
 		if repo_config.writable:
 			writable_location = repo_config.location
@@ -1149,10 +1136,37 @@ def egencache_main(args):
 	if options.update_changelogs:
 		gen_clogs = GenChangeLogs(portdb,
 			changelog_output=options.changelog_output,
-			changelog_reversed=options.changelog_reversed)
-		gen_clogs.run()
+			changelog_reversed=options.changelog_reversed,
+			max_jobs=options.jobs,
+			max_load=options.load_average)
+		signum = gen_clogs.run()
+		if signum is not None:
+			sys.exit(128 + signum)
 		ret.append(gen_clogs.returncode)
 
+	if options.update_manifests:
+
+		cp_iter = None
+		if atoms:
+			cp_iter = iter(atoms)
+
+		event_loop = global_event_loop()
+		scheduler = ManifestScheduler(portdb, cp_iter=cp_iter,
+			gpg_cmd=gpg_cmd, gpg_vars=gpg_vars,
+			force_sign_key=force_sign_key,
+			max_jobs=options.jobs,
+			max_load=options.load_average,
+			event_loop=event_loop)
+
+		signum = run_main_scheduler(scheduler)
+		if signum is not None:
+			sys.exit(128 + signum)
+
+		if options.tolerant:
+			ret.append(os.EX_OK)
+		else:
+			ret.append(scheduler.returncode)
+
 	if options.write_timestamp:
 		timestamp_path = os.path.join(repo_path, 'metadata', 'timestamp.chk')
 		try:
diff --git a/bin/install-qa-check.d/60openrc b/bin/install-qa-check.d/60openrc
index 9b7fc6d99..1e56b2f37 100644
--- a/bin/install-qa-check.d/60openrc
+++ b/bin/install-qa-check.d/60openrc
@@ -24,7 +24,7 @@ openrc_check() {
 			for i in "${ED}${d}"/* ; do
 				[[ -e ${i} ]] || continue
 				[[ -L ${i} ]] && continue
-				f=$("${checkbashisms}" -f "${i}" 2>&1)
+				f=$("${checkbashisms}" -n -f "${i}" 2>&1)
 				[[ $? != 0 && -n ${f} ]] || continue
 				eqawarn "QA Notice: shell script appears to use non-POSIX feature(s):"
 				while read -r ;
diff --git a/bin/save-ebuild-env.sh b/bin/save-ebuild-env.sh
index 477ed28ed..17e4cdcbb 100644
--- a/bin/save-ebuild-env.sh
+++ b/bin/save-ebuild-env.sh
@@ -51,7 +51,7 @@ __save_ebuild_env() {
 		__dump_trace die \
 		__quiet_mode __vecho __elog_base eqawarn elog \
 		einfo einfon ewarn eerror ebegin __eend eend KV_major \
-		KV_minor KV_micro KV_to_int get_KV __1 __1 has \
+		KV_minor KV_micro KV_to_int get_KV has \
 		__has_phase_defined_up_to \
 		hasv hasq __qa_source __qa_call \
 		addread addwrite adddeny addpredict __sb_append_var \
diff --git a/man/portage.5 b/man/portage.5
index 8e2be4f63..c9e70a0e3 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1,4 +1,4 @@
-.TH "PORTAGE" "5" "Feb 2015" "Portage VERSION" "Portage"
+.TH "PORTAGE" "5" "Nov 2015" "Portage VERSION" "Portage"
 .SH NAME
 portage \- the heart of Gentoo
 .SH "DESCRIPTION"
@@ -968,6 +968,11 @@ Specifies CVS repository.
 Specifies clone depth to use for DVCS repositories. Defaults to 1 (only
 the newest commit). If set to 0, the depth is unlimited.
 .TP
+.B sync\-hooks\-only\-on\-change
+If set to true, then sync of a given repository will not trigger postsync
+hooks unless hooks would have executed for a master repository or the
+repository has changed since the previous sync operation.
+.TP
 .B sync\-type
 Specifies type of synchronization performed by `emerge \-\-sync`.
 .br
diff --git a/misc/emerge-delta-webrsync b/misc/emerge-delta-webrsync
index 05a0ac721..f2dc8220c 100755
--- a/misc/emerge-delta-webrsync
+++ b/misc/emerge-delta-webrsync
@@ -754,7 +754,7 @@ if [ "$verified" == "1" ]; then
 	rm -f "${TMPDIR}/portage-${final_date}.tar"
 else
 	echo "recompressing."
-	bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar.bz2"
+	bzip2 -v9 "${TMPDIR}/portage-${final_date}.tar"
 fi
 
 echo "verifying generated tarball"
diff --git a/pym/portage/emaint/modules/sync/sync.py b/pym/portage/emaint/modules/sync/sync.py
index 57c779d31..15d63e2c8 100644
--- a/pym/portage/emaint/modules/sync/sync.py
+++ b/pym/portage/emaint/modules/sync/sync.py
@@ -233,15 +233,17 @@ class SyncRepos(object):
 		retvals = sync_scheduler.retvals
 		msgs.extend(sync_scheduler.msgs)
 
-		# run the post_sync_hook one last time for
-		# run only at sync completion hooks
-		rcode = sync_manager.perform_post_sync_hook('')
 		if retvals:
 			msgs.extend(self.rmessage(retvals, 'sync'))
 		else:
 			msgs.extend(self.rmessage([('None', os.EX_OK)], 'sync'))
-		if rcode:
-			msgs.extend(self.rmessage([('None', rcode)], 'post-sync'))
+
+		# run the post_sync_hook one last time for
+		# run only at sync completion hooks
+		if sync_scheduler.global_hooks_enabled:
+			rcode = sync_manager.perform_post_sync_hook('')
+			if rcode:
+				msgs.extend(self.rmessage([('None', rcode)], 'post-sync'))
 
 		# Reload the whole config.
 		portage._sync_mode = False
@@ -339,6 +341,8 @@ class SyncScheduler(AsyncScheduler):
 				if master.name in selected_repo_names:
 					self._repo_map[master.name] = master
 					self._sync_graph.add(master.name, repo.name)
+		self._complete_graph = self._sync_graph.copy()
+		self._hooks_repos = set()
 		self._update_leaf_nodes()
 
 	def _task_exit(self, task):
@@ -347,9 +351,13 @@ class SyncScheduler(AsyncScheduler):
 		more leaf nodes.
 		'''
 		self._running_tasks.discard(task)
+		# Set hooks_enabled = True by default, in order to ensure
+		# that hooks will be called in a backward-compatible manner
+		# even if all sync tasks have failed.
+		hooks_enabled = True
 		returncode = task.returncode
 		if task.returncode == os.EX_OK:
-			returncode, message, updatecache_flg = task.result
+			returncode, message, updatecache_flg, hooks_enabled = task.result
 			if message:
 				self.msgs.append(message)
 		repo = task.kwargs['repo'].name
@@ -357,8 +365,38 @@ class SyncScheduler(AsyncScheduler):
 		self.retvals.append((repo, returncode))
 		self._sync_graph.remove(repo)
 		self._update_leaf_nodes()
+		if hooks_enabled:
+			self._hooks_repos.add(repo)
 		super(SyncScheduler, self)._task_exit(self)
 
+	def _master_hooks(self, repo_name):
+		"""
+		@param repo_name: a repo name
+		@type repo_name: str
+		@return: True if hooks would have been executed for any master
+			repositories of the given repo, False otherwise
+		@rtype: bool
+		"""
+		traversed_nodes = set()
+		node_stack = [repo_name]
+		while node_stack:
+			node = node_stack.pop()
+			if node in self._hooks_repos:
+				return True
+			if node not in traversed_nodes:
+				traversed_nodes.add(node)
+				node_stack.extend(self._complete_graph.child_nodes(node))
+		return False
+
+	@property
+	def global_hooks_enabled(self):
+		"""
+		@return: True if repo.postsync.d hooks would have been executed
+			for any repositories.
+		@rtype: bool
+		"""
+		return bool(self._hooks_repos)
+
 	def _update_leaf_nodes(self):
 		'''
 		Populate self._leaf_nodes with current leaves from
@@ -389,9 +427,10 @@ class SyncScheduler(AsyncScheduler):
 		self._running_repos.add(node)
 		self._update_leaf_nodes()
 
-		task = self._sync_manager.async(
-			self._emerge_config, self._repo_map[node])
-		return task
+		return self._sync_manager.async(
+			emerge_config=self._emerge_config,
+			repo=self._repo_map[node],
+			master_hooks=self._master_hooks(node))
 
 	def _can_add_job(self):
 		'''
diff --git a/pym/portage/repository/config.py b/pym/portage/repository/config.py
index 1060bc7c9..ffb454445 100644
--- a/pym/portage/repository/config.py
+++ b/pym/portage/repository/config.py
@@ -87,7 +87,7 @@ class RepoConfig(object):
 		'main_repo', 'manifest_hashes', 'masters', 'missing_repo_name',
 		'name', 'portage1_profiles', 'portage1_profiles_compat', 'priority',
 		'profile_formats', 'sign_commit', 'sign_manifest',
-		'sync_depth',
+		'sync_depth', 'sync_hooks_only_on_change',
 		'sync_type', 'sync_umask', 'sync_uri', 'sync_user', 'thin_manifest',
 		'update_changelog', 'user_location', '_eapis_banned',
 		'_eapis_deprecated', '_masters_orig', 'module_specific_options',
@@ -175,6 +175,8 @@ class RepoConfig(object):
 		self.auto_sync = auto_sync
 
 		self.sync_depth = repo_opts.get('sync-depth')
+		self.sync_hooks_only_on_change = repo_opts.get(
+			'sync-hooks-only-on-change', 'false').lower() == 'true'
 
 		self.module_specific_options = {}
 
@@ -506,7 +508,7 @@ class RepoConfigLoader(object):
 						# repos.conf is allowed to override.
 						for k in ('aliases', 'auto_sync', 'eclass_overrides',
 							'force', 'masters', 'priority',
-							'sync_depth',
+							'sync_depth', 'sync_hooks_only_on_change',
 							'sync_type', 'sync_umask', 'sync_uri', 'sync_user',
 							'module_specific_options'):
 							v = getattr(repos_conf_opts, k, None)
diff --git a/pym/portage/sync/controller.py b/pym/portage/sync/controller.py
index e8132c2ba..4595293c6 100644
--- a/pym/portage/sync/controller.py
+++ b/pym/portage/sync/controller.py
@@ -114,16 +114,17 @@ class SyncManager(object):
 			return desc
 		return []
 
-	def async(self, emerge_config=None, repo=None):
+	def async(self, emerge_config=None, repo=None, master_hooks=True):
 		self.emerge_config = emerge_config
 		self.settings, self.trees, self.mtimedb = emerge_config
 		self.xterm_titles = "notitles" not in self.settings.features
 		self.portdb = self.trees[self.settings['EROOT']]['porttree'].dbapi
 		return SyncRepo(sync_task=AsyncFunction(target=self.sync,
-			kwargs=dict(emerge_config=emerge_config, repo=repo)),
+			kwargs=dict(emerge_config=emerge_config, repo=repo,
+			master_hooks=master_hooks)),
 			sync_callback=self._sync_callback)
 
-	def sync(self, emerge_config=None, repo=None):
+	def sync(self, emerge_config=None, repo=None, master_hooks=True):
 		self.callback = None
 		self.repo = repo
 		self.exitcode = 1
@@ -156,9 +157,14 @@ class SyncManager(object):
 		taskmaster = TaskHandler(callback=self.do_callback)
 		taskmaster.run_tasks(tasks, func, status, options=task_opts)
 
-		self.perform_post_sync_hook(repo.name, repo.sync_uri, repo.location)
+		hooks_enabled = False
+		if (master_hooks or self.updatecache_flg or
+			not repo.sync_hooks_only_on_change):
+			hooks_enabled = True
+			self.perform_post_sync_hook(
+				repo.name, repo.sync_uri, repo.location)
 
-		return self.exitcode, None, self.updatecache_flg
+		return self.exitcode, None, self.updatecache_flg, hooks_enabled
 
 
 	def do_callback(self, result):
@@ -328,7 +334,7 @@ class SyncManager(object):
 		exitcode = proc.returncode
 		updatecache_flg = False
 		if proc.returncode == os.EX_OK:
-			exitcode, message, updatecache_flg = proc.result
+			exitcode, message, updatecache_flg, hooks_enabled = proc.result
 
 		if updatecache_flg and "metadata-transfer" not in self.settings.features:
 			updatecache_flg = False
diff --git a/pym/portage/sync/modules/git/git.py b/pym/portage/sync/modules/git/git.py
index c14782c38..179c0dea8 100644
--- a/pym/portage/sync/modules/git/git.py
+++ b/pym/portage/sync/modules/git/git.py
@@ -2,6 +2,7 @@
 # Distributed under the terms of the GNU General Public License v2
 
 import logging
+import subprocess
 
 import portage
 from portage import os
@@ -81,6 +82,10 @@ class GitSync(NewBase):
 		git_cmd = "%s pull%s" % (self.bin_command, git_cmd_opts)
 		writemsg_level(git_cmd + "\n")
 
+		rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
+		previous_rev = subprocess.check_output(rev_cmd,
+			cwd=portage._unicode_encode(self.repo.location))
+
 		exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
 				portage._shell_quote(self.repo.location), git_cmd),
 			**portage._native_kwargs(self.spawn_kwargs))
@@ -89,4 +94,8 @@ class GitSync(NewBase):
 			self.logger(self.xterm_titles, msg)
 			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
 			return (exitcode, False)
-		return (os.EX_OK, True)
+
+		current_rev = subprocess.check_output(rev_cmd,
+			cwd=portage._unicode_encode(self.repo.location))
+
+		return (os.EX_OK, current_rev != previous_rev)
diff --git a/pym/portage/sync/modules/rsync/rsync.py b/pym/portage/sync/modules/rsync/rsync.py
index 8ae8a5caf..e0f76b331 100644
--- a/pym/portage/sync/modules/rsync/rsync.py
+++ b/pym/portage/sync/modules/rsync/rsync.py
@@ -112,10 +112,10 @@ class RsyncSync(NewBase):
 		if syncuri.startswith("file://"):
 			self.proto = "file"
 			dosyncuri = syncuri[6:]
-			is_synced, exitcode = self._do_rsync(
+			is_synced, exitcode, updatecache_flg = self._do_rsync(
 				dosyncuri, timestamp, opts)
 			self._process_exitcode(exitcode, dosyncuri, out, 1)
-			return (exitcode, exitcode == os.EX_OK)
+			return (exitcode, updatecache_flg)
 
 		retries=0
 		try:
@@ -138,7 +138,7 @@ class RsyncSync(NewBase):
 		else:
 			# getaddrinfo needs the brackets stripped
 			getaddrinfo_host = hostname[1:-1]
-		updatecache_flg=True
+		updatecache_flg = False
 		all_rsync_opts = set(self.rsync_opts)
 		all_rsync_opts.update(self.extra_rsync_opts)
 
@@ -240,7 +240,8 @@ class RsyncSync(NewBase):
 			if dosyncuri.startswith('ssh://'):
 				dosyncuri = dosyncuri[6:].replace('/', ':/', 1)
 
-			is_synced, exitcode = self._do_rsync(dosyncuri, timestamp, opts)
+			is_synced, exitcode, updatecache_flg = self._do_rsync(
+				dosyncuri, timestamp, opts)
 			if is_synced:
 				break
 
@@ -251,7 +252,6 @@ class RsyncSync(NewBase):
 			else:
 				# over retries
 				# exit loop
-				updatecache_flg=False
 				exitcode = EXCEEDED_MAX_RETRIES
 				break
 		self._process_exitcode(exitcode, dosyncuri, out, maxretries)
@@ -382,6 +382,7 @@ class RsyncSync(NewBase):
 
 
 	def _do_rsync(self, syncuri, timestamp, opts):
+		updatecache_flg = False
 		is_synced = False
 		if timestamp != 0 and "--quiet" not in opts:
 			print(">>> Checking server timestamp ...")
@@ -489,7 +490,7 @@ class RsyncSync(NewBase):
 				print(">>> In order to force sync, remove '%s'." % self.servertimestampfile)
 				print(">>>")
 				print()
-				return is_synced, exitcode
+				return is_synced, exitcode, updatecache_flg
 			elif (servertimestamp != 0) and (servertimestamp < timestamp):
 				self.logger(self.xterm_titles,
 					">>> Server out of date: %s" % syncuri)
@@ -543,6 +544,8 @@ class RsyncSync(NewBase):
 							os.unlink(self.servertimestampfile)
 						except OSError:
 							pass
+					else:
+						updatecache_flg = True
 
 				if exitcode in [0,1,3,4,11,14,20,21]:
 					is_synced = True
@@ -554,4 +557,4 @@ class RsyncSync(NewBase):
 			# --prune-empty-directories.  Retry for a server that supports
 			# at least rsync protocol version 29 (>=rsync-2.6.4).
 			pass
-		return is_synced, exitcode
+		return is_synced, exitcode, updatecache_flg
diff --git a/pym/portage/util/_async/AsyncFunction.py b/pym/portage/util/_async/AsyncFunction.py
index b6142a214..40f6c5e75 100644
--- a/pym/portage/util/_async/AsyncFunction.py
+++ b/pym/portage/util/_async/AsyncFunction.py
@@ -15,7 +15,10 @@ class AsyncFunction(ForkProcess):
 	"result" attribute after the forked process has exited.
 	"""
 
-	__slots__ = ('args', 'kwargs', 'result', 'target',
+	# NOTE: This class overrides the meaning of the SpawnProcess 'args'
+	# attribute, and uses it to hold the positional arguments for the
+	# 'target' function.
+	__slots__ = ('kwargs', 'result', 'target',
 		'_async_func_reader', '_async_func_reader_pw')
 
 	def _start(self):