aboutsummaryrefslogtreecommitdiff
blob: 7df4b6d616487b0e161c9e761b67005099029bf9 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
# Copyright 2005-2018 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2

import io
import logging
import subprocess

import portage
from portage import os
from portage.util import writemsg_level, shlex_split
from portage.util.futures import asyncio
from portage.output import create_color_func, EOutput
good = create_color_func("GOOD")
bad = create_color_func("BAD")
warn = create_color_func("WARN")
from portage.sync.syncbase import NewBase

try:
	from gemato.exceptions import GematoException
	import gemato.openpgp
except ImportError:
	gemato = None


class GitSync(NewBase):
	'''Git sync class'''

	short_desc = "Perform sync operations on git based repositories"

	@staticmethod
	def name():
		return "GitSync"


	def __init__(self):
		NewBase.__init__(self, "git", portage.const.GIT_PACKAGE_ATOM)


	def exists(self, **kwargs):
		'''Tests whether the repo actually exists'''
		return os.path.exists(os.path.join(self.repo.location, '.git'))


	def new(self, **kwargs):
		'''Do the initial clone of the repository'''
		if kwargs:
			self._kwargs(kwargs)
		if not self.has_bin:
			return (1, False)
		try:
			if not os.path.exists(self.repo.location):
				os.makedirs(self.repo.location)
				self.logger(self.xterm_titles,
					'Created new directory %s' % self.repo.location)
		except IOError:
			return (1, False)

		sync_uri = self.repo.sync_uri
		if sync_uri.startswith("file://"):
			sync_uri = sync_uri[7:]

		git_cmd_opts = ""
		if self.repo.module_specific_options.get('sync-git-env'):
			shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
			env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
			self.spawn_kwargs['env'].update(env)

		if self.repo.module_specific_options.get('sync-git-clone-env'):
			shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-clone-env'])
			clone_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
			self.spawn_kwargs['env'].update(clone_env)

		if self.settings.get("PORTAGE_QUIET") == "1":
			git_cmd_opts += " --quiet"
		if self.repo.clone_depth is not None:
			if self.repo.clone_depth != 0:
				git_cmd_opts += " --depth %d" % self.repo.clone_depth
		elif self.repo.sync_depth is not None:
			if self.repo.sync_depth != 0:
				git_cmd_opts += " --depth %d" % self.repo.sync_depth
		else:
			# default
			git_cmd_opts += " --depth 1"

		if self.repo.module_specific_options.get('sync-git-clone-extra-opts'):
			git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-clone-extra-opts']
		git_cmd = "%s clone%s %s ." % (self.bin_command, git_cmd_opts,
			portage._shell_quote(sync_uri))
		writemsg_level(git_cmd + "\n")

		exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
				portage._shell_quote(self.repo.location), git_cmd),
			**self.spawn_kwargs)
		if exitcode != os.EX_OK:
			msg = "!!! git clone error in %s" % self.repo.location
			self.logger(self.xterm_titles, msg)
			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
			return (exitcode, False)
		if not self.verify_head():
			return (1, False)
		return (os.EX_OK, True)


	def update(self):
		''' Update existing git repository, and ignore the syncuri. We are
		going to trust the user and assume that the user is in the branch
		that he/she wants updated. We'll let the user manage branches with
		git directly.
		'''
		if not self.has_bin:
			return (1, False)
		git_cmd_opts = ""
		quiet = self.settings.get("PORTAGE_QUIET") == "1"
		if self.repo.module_specific_options.get('sync-git-env'):
			shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-env'])
			env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
			self.spawn_kwargs['env'].update(env)

		if self.repo.module_specific_options.get('sync-git-pull-env'):
			shlexed_env = shlex_split(self.repo.module_specific_options['sync-git-pull-env'])
			pull_env = dict((k, v) for k, _, v in (assignment.partition('=') for assignment in shlexed_env) if k)
			self.spawn_kwargs['env'].update(pull_env)

		if self.settings.get("PORTAGE_QUIET") == "1":
			git_cmd_opts += " --quiet"
		if self.repo.module_specific_options.get('sync-git-pull-extra-opts'):
			git_cmd_opts += " %s" % self.repo.module_specific_options['sync-git-pull-extra-opts']

		try:
			remote_branch = portage._unicode_decode(
				subprocess.check_output([self.bin_command, 'rev-parse',
				'--abbrev-ref', '--symbolic-full-name', '@{upstream}'],
				cwd=portage._unicode_encode(self.repo.location))).rstrip('\n')
		except subprocess.CalledProcessError as e:
			msg = "!!! git rev-parse error in %s" % self.repo.location
			self.logger(self.xterm_titles, msg)
			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
			return (e.returncode, False)

		shallow = self.repo.sync_depth is not None and self.repo.sync_depth != 0
		if shallow:
			git_cmd_opts += " --depth %d" % self.repo.sync_depth

			# For shallow fetch, unreachable objects may need to be pruned
			# manually, in order to prevent automatic git gc calls from
			# eventually failing (see bug 599008).
			gc_cmd = ['git', '-c', 'gc.autodetach=false', 'gc', '--auto']
			if quiet:
				gc_cmd.append('--quiet')
			exitcode = portage.process.spawn(gc_cmd,
				cwd=portage._unicode_encode(self.repo.location),
				**self.spawn_kwargs)
			if exitcode != os.EX_OK:
				msg = "!!! git gc error in %s" % self.repo.location
				self.logger(self.xterm_titles, msg)
				writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
				return (exitcode, False)

		git_cmd = "%s fetch %s%s" % (self.bin_command,
			remote_branch.partition('/')[0], git_cmd_opts)

		writemsg_level(git_cmd + "\n")

		rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
		previous_rev = subprocess.check_output(rev_cmd,
			cwd=portage._unicode_encode(self.repo.location))

		exitcode = portage.process.spawn_bash("cd %s ; exec %s" % (
				portage._shell_quote(self.repo.location), git_cmd),
			**self.spawn_kwargs)

		if exitcode != os.EX_OK:
			msg = "!!! git fetch error in %s" % self.repo.location
			self.logger(self.xterm_titles, msg)
			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
			return (exitcode, False)

		if not self.verify_head(revision='refs/remotes/%s' % remote_branch):
			return (1, False)

		if shallow:
			# Since the default merge strategy typically fails when
			# the depth is not unlimited, `git reset --merge`.
			merge_cmd = [self.bin_command, 'reset', '--merge']
		else:
			merge_cmd = [self.bin_command, 'merge']
		merge_cmd.append('refs/remotes/%s' % remote_branch)
		if quiet:
			merge_cmd.append('--quiet')
		exitcode = portage.process.spawn(merge_cmd,
			cwd=portage._unicode_encode(self.repo.location),
			**self.spawn_kwargs)

		if exitcode != os.EX_OK:
			msg = "!!! git merge error in %s" % self.repo.location
			self.logger(self.xterm_titles, msg)
			writemsg_level(msg + "\n", level=logging.ERROR, noiselevel=-1)
			return (exitcode, False)

		current_rev = subprocess.check_output(rev_cmd,
			cwd=portage._unicode_encode(self.repo.location))

		return (os.EX_OK, current_rev != previous_rev)

	def verify_head(self, revision='-1'):
		if (self.repo.module_specific_options.get(
				'sync-git-verify-commit-signature', 'false') != 'true'):
			return True

		if self.repo.sync_openpgp_key_path is not None:
			if gemato is None:
				writemsg_level("!!! Verifying against specified key requires gemato-11.0+ installed\n",
					level=logging.ERROR, noiselevel=-1)
				return False
			openpgp_env = gemato.openpgp.OpenPGPEnvironment()
		else:
			openpgp_env = None

		try:
			out = EOutput()
			env = None
			if openpgp_env is not None:
				try:
					out.einfo('Using keys from %s' % (self.repo.sync_openpgp_key_path,))
					with io.open(self.repo.sync_openpgp_key_path, 'rb') as f:
						openpgp_env.import_key(f)
					self._refresh_keys(openpgp_env)
				except (GematoException, asyncio.TimeoutError) as e:
					writemsg_level("!!! Verification impossible due to keyring problem:\n%s\n"
							% (e,),
							level=logging.ERROR, noiselevel=-1)
					return False

				env = os.environ.copy()
				env['GNUPGHOME'] = openpgp_env.home

			rev_cmd = [self.bin_command, "log", "-n1", "--pretty=format:%G?", revision]
			try:
				status = (portage._unicode_decode(
					subprocess.check_output(rev_cmd,
						cwd=portage._unicode_encode(self.repo.location),
						env=env))
					.strip())
			except subprocess.CalledProcessError:
				return False

			if status == 'G':  # good signature is good
				out.einfo('Trusted signature found on top commit')
				return True
			elif status == 'U':  # untrusted
				out.ewarn('Top commit signature is valid but not trusted')
				return True
			else:
				if status == 'B':
					expl = 'bad signature'
				elif status == 'X':
					expl = 'expired signature'
				elif status == 'Y':
					expl = 'expired key'
				elif status == 'R':
					expl = 'revoked key'
				elif status == 'E':
					expl = 'unable to verify signature (missing key?)'
				elif status == 'N':
					expl = 'no signature'
				else:
					expl = 'unknown issue'
				out.eerror('No valid signature found: %s' % (expl,))
				return False
		finally:
			if openpgp_env is not None:
				openpgp_env.close()

	def retrieve_head(self, **kwargs):
		'''Get information about the head commit'''
		if kwargs:
			self._kwargs(kwargs)
		if self.bin_command is None:
			# return quietly so that we don't pollute emerge --info output
			return (1, False)
		rev_cmd = [self.bin_command, "rev-list", "--max-count=1", "HEAD"]
		try:
			ret = (os.EX_OK,
				portage._unicode_decode(subprocess.check_output(rev_cmd,
				cwd=portage._unicode_encode(self.repo.location))))
		except subprocess.CalledProcessError:
			ret = (1, False)
		return ret