repoman: add --include-profiles=PROFILES

repoman slows down ~linearly with amount of profiles being scanned.
In case of amd64 we have 28 stable profiles.

To speed up processing and fit into time budged of various CIs we can
split the work across different processes that handle different profiles.

Example benchmark on ::haskell overlay:
    $ ./repoman full --include-arches=amd64
    ~65 minutes
    $ ./repoman full --include-profiles=default/linux/amd64/17.0
    ~4 minutes
This allows for a crude sharding of work across processes and allows for
cheap tree-wide scans for early failures.

Bug: https://bugs.gentoo.org/700456
Signed-off-by: Sergei Trofimovich <slyfox@gentoo.org>
Signed-off-by: Zac Medico <zmedico@gentoo.org>

6 files changed, 26 insertions, 3 deletions

diff --git a/repoman/lib/repoman/actions.py b/repoman/lib/repoman/actions.py
index 1c9989a72..92d4d4e94 100644
--- a/repoman/lib/repoman/actions.py
+++ b/repoman/lib/repoman/actions.py
@@ -412,6 +412,10 @@ the whole commit message to abort.
 			report_options.append(
 				"--include-arches=\"%s\"" %
 				" ".join(sorted(self.scanner.include_arches)))
+		if self.scanner.include_profiles is not None:
+			report_options.append(
+				"--include-profiles=\"%s\"" %
+				" ".join(sorted(self.scanner.include_profiles)))
 
 		if portage_version is None:
 			sys.stderr.write("Failed to insert portage version in message!\n")
diff --git a/repoman/lib/repoman/argparser.py b/repoman/lib/repoman/argparser.py
index fa0e6ff90..670a0e91d 100644
--- a/repoman/lib/repoman/argparser.py
+++ b/repoman/lib/repoman/argparser.py
@@ -165,6 +165,13 @@ def parse_args(argv, repoman_default_opts):
 			'filter the selection of profiles for dependency checks'))
 
 	parser.add_argument(
+		'--include-profiles',
+		dest='include_profiles', metavar='PROFILES', action='append',
+		help=(
+			'A space separated list of profiles used to '
+			'define the selection of profiles for dependency checks'))
+
+	parser.add_argument(
 		'-d', '--include-dev', dest='include_dev', action='store_true',
 		default=False,
 		help='include dev profiles in dependency checks')
diff --git a/repoman/lib/repoman/modules/scan/depend/__init__.py b/repoman/lib/repoman/modules/scan/depend/__init__.py
index c3cc0ddeb..9068760bb 100644
--- a/repoman/lib/repoman/modules/scan/depend/__init__.py
+++ b/repoman/lib/repoman/modules/scan/depend/__init__.py
@@ -19,7 +19,8 @@ module_spec = {
 			'func_desc': {
 			},
 			'mod_kwargs': ['qatracker', 'portdb', 'profiles', 'options',
-				'repo_metadata', 'repo_settings', 'include_arches', 'caches',
+				'repo_metadata', 'repo_settings', 'include_arches',
+				'include_profiles', 'caches',
 				'repoman_incrementals', 'env', 'have', 'dev_keywords'
 			],
 			'func_kwargs': {
diff --git a/repoman/lib/repoman/modules/scan/depend/profile.py b/repoman/lib/repoman/modules/scan/depend/profile.py
index d980f4eca..39d8b550c 100644
--- a/repoman/lib/repoman/modules/scan/depend/profile.py
+++ b/repoman/lib/repoman/modules/scan/depend/profile.py
@@ -33,6 +33,7 @@ class ProfileDependsChecks(ScanBase):
 		@param options: cli options
 		@param repo_settings: repository settings instance
 		@param include_arches: set
+		@param include_profiles: set
 		@param caches: dictionary of our caches
 		@param repoman_incrementals: tuple
 		@param env: the environment
@@ -46,6 +47,7 @@ class ProfileDependsChecks(ScanBase):
 		self.options = kwargs.get('options')
 		self.repo_settings = kwargs.get('repo_settings')
 		self.include_arches = kwargs.get('include_arches')
+		self.include_profiles = kwargs.get('include_profiles')
 		self.caches = kwargs.get('caches')
 		self.repoman_incrementals = kwargs.get('repoman_incrementals')
 		self.env = kwargs.get('env')
@@ -81,8 +83,11 @@ class ProfileDependsChecks(ScanBase):
 				if arch not in self.include_arches:
 					continue
 
-			relevant_profiles.extend(
-				(keyword, groups, prof) for prof in self.profiles[arch])
+			for prof in self.profiles[arch]:
+				if self.include_profiles is not None:
+					if prof.sub_path not in self.include_profiles:
+						continue
+				relevant_profiles.append((keyword, groups, prof))
 
 		relevant_profiles.sort(key=sort_key)
 
diff --git a/repoman/lib/repoman/scanner.py b/repoman/lib/repoman/scanner.py
index 1b3242a51..06234b0ad 100644
--- a/repoman/lib/repoman/scanner.py
+++ b/repoman/lib/repoman/scanner.py
@@ -164,6 +164,10 @@ class Scanner(object):
 		if self.options.include_arches:
 			self.include_arches = set()
 			self.include_arches.update(*[x.split() for x in self.options.include_arches])
+		self.include_profiles = None
+		if self.options.include_profiles:
+			self.include_profiles = set()
+			self.include_profiles.update(*[x.split() for x in self.options.include_profiles])
 
 		# Disable the "self.modules['Ebuild'].notadded" check when not in commit mode and
 		# running `svn status` in every package dir will be too expensive.
@@ -190,6 +194,7 @@ class Scanner(object):
 			"repo_metadata": self.repo_metadata,
 			"profiles": profiles,
 			"include_arches": self.include_arches,
+			"include_profiles": self.include_profiles,
 			"caches": self.caches,
 			"repoman_incrementals": self.repoman_incrementals,
 			"env": self.env,
diff --git a/repoman/lib/repoman/tests/simple/test_simple.py b/repoman/lib/repoman/tests/simple/test_simple.py
index b0cc43297..c86a183f9 100644
--- a/repoman/lib/repoman/tests/simple/test_simple.py
+++ b/repoman/lib/repoman/tests/simple/test_simple.py
@@ -202,6 +202,7 @@ class SimpleRepomanTestCase(TestCase):
 			("", git_cmd + ("add", ".")),
 			("", git_cmd + ("commit", "-a", "-m", "add whole repo")),
 			("", repoman_cmd + ("full", "-d")),
+			("", repoman_cmd + ("full", "--include-profiles", "default/linux/x86/test_profile")),
 			("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")),
 			("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")),
 			("", repoman_cmd + ("commit", "-m", "cat/pkg: bump to version 2")),