[Notes] [Git][BuildStream/buildstream][537-mirror-fallback-does-not-work-for-git] 3 commits: tests: Test that fetching passes when upstream is absent

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch 537-mirror-fallback-does-not-work-for-git at BuildStream / buildstream

Commits:

• c945046d
  by Jonathan Maw at 2018-08-06T12:31:50Z

  tests: Test that fetching passes when upstream is absent
  

• d3db49ec
  by Jonathan Maw at 2018-08-06T18:26:13Z

  source.py: Fix documentation on get_source_fetchers
  

• b49d16f3
  by Jonathan Maw at 2018-08-06T18:28:46Z

  git.py: Use the configured list of submodules when fetching with aliases
  
  When the source's URL uses an alias (i.e. Source.translate_url()
  actually changes the URL), we use the list of submodules defined in the
  config as the list of submodules used.
  This is because the source would otherwise have to fetch the main repo
  to know which submodules it actually has.
  This is inconvenient when using mirroring, because when the upstream
  repo is unreachable it'll fail when it doesn't have to.
  
  GitMirrors can now be instantiated without having their path and ref
  defined, provided the 'parent' mirror is defined. It will retrieve the
  path and ref as-needed instead.
  
  If the source's URL does not use an alias, it will behave as normally.
  

3 changed files:

• buildstream/plugins/sources/git.py
• buildstream/source.py
• tests/frontend/mirror.py

Changes:

buildstream/plugins/sources/git.py

---

...	...	@@ -91,16 +91,18 @@ GIT_MODULES = '.gitmodules'
91	91	#
92	92	class GitMirror(SourceFetcher):
93	93
94		- def __init__(self, source, path, url, ref):
	94	+ def __init__(self, source, path, url, ref, *, parent=None):
95	95
96	96	super().__init__()
97	97	self.source = source
98		- self.path = path
	98	+ self.parent = parent
99	99	self.url = url
100		- self.ref = ref
101	100	self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
102	101	self.mark_download_url(url)
103	102
	103	+ self._path = path
	104	+ self._ref = ref
	105	+
104	106	# Ensures that the mirror exists
105	107	def ensure(self, alias_override=None):
106	108
...	...	@@ -167,7 +169,8 @@ class GitMirror(SourceFetcher):
167	169	self.assert_ref()
168	170
169	171	def has_ref(self):
170		- if not self.ref:
	172	+ ref = self.get_ref()
	173	+ if not ref:
171	174	return False
172	175
173	176	# If the mirror doesnt exist, we also dont have the ref
...	...	@@ -175,13 +178,13 @@ class GitMirror(SourceFetcher):
175	178	return False
176	179
177	180	# Check if the ref is really there
178		- rc = self.source.call([self.source.host_git, 'cat-file', '-t', self.ref], cwd=self.mirror)
	181	+ rc = self.source.call([self.source.host_git, 'cat-file', '-t', ref], cwd=self.mirror)
179	182	return rc == 0
180	183
181	184	def assert_ref(self):
182	185	if not self.has_ref():
183	186	raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
184		- .format(self.source, self.ref, self.url))
	187	+ .format(self.source, self.get_ref(), self.url))
185	188
186	189	def latest_commit(self, tracking):
187	190	_, output = self.source.check_output(
...	...	@@ -191,7 +194,8 @@ class GitMirror(SourceFetcher):
191	194	return output.rstrip('\n')
192	195
193	196	def stage(self, directory):
194		- fullpath = os.path.join(directory, self.path)
	197	+ fullpath = os.path.join(directory, self.get_path())
	198	+ ref = self.get_ref()
195	199
196	200	# Using --shared here avoids copying the objects into the checkout, in any
197	201	# case we're just checking out a specific commit and then removing the .git/
...	...	@@ -200,16 +204,17 @@ class GitMirror(SourceFetcher):
200	204	fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
201	205	fail_temporarily=True)
202	206
203		- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
204		- fail="Failed to checkout git ref {}".format(self.ref),
	207	+ self.source.call([self.source.host_git, 'checkout', '--force', ref],
	208	+ fail="Failed to checkout git ref {}".format(ref),
205	209	cwd=fullpath)
206	210
207	211	# Remove .git dir
208	212	shutil.rmtree(os.path.join(fullpath, ".git"))
209	213
210	214	def init_workspace(self, directory):
211		- fullpath = os.path.join(directory, self.path)
	215	+ fullpath = os.path.join(directory, self.get_path())
212	216	url = self.source.translate_url(self.url)
	217	+ ref = self.get_ref()
213	218
214	219	self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
215	220	fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
...	...	@@ -219,13 +224,13 @@ class GitMirror(SourceFetcher):
219	224	fail='Failed to add remote origin "{}"'.format(url),
220	225	cwd=fullpath)
221	226
222		- self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
223		- fail="Failed to checkout git ref {}".format(self.ref),
	227	+ self.source.call([self.source.host_git, 'checkout', '--force', ref],
	228	+ fail="Failed to checkout git ref {}".format(ref),
224	229	cwd=fullpath)
225	230
226		- # List the submodules (path/url tuples) present at the given ref of this repo
227		- def submodule_list(self):
228		- modules = "{}:{}".format(self.ref, GIT_MODULES)
	231	+ def _read_gitmodules(self):
	232	+ ref = self.get_ref()
	233	+ modules = "{}:{}".format(ref, GIT_MODULES)
229	234	exit_code, output = self.source.check_output(
230	235	[self.source.host_git, 'show', modules], cwd=self.mirror)
231	236
...	...	@@ -236,7 +241,7 @@ class GitMirror(SourceFetcher):
236	241	elif exit_code != 0:
237	242	raise SourceError(
238	243	"{plugin}: Failed to show gitmodules at ref {ref}".format(
239		- plugin=self, ref=self.ref))
	244	+ plugin=self, ref=ref))
240	245
241	246	content = '\n'.join([l.strip() for l in output.splitlines()])
242	247
...	...	@@ -247,16 +252,21 @@ class GitMirror(SourceFetcher):
247	252	for section in parser.sections():
248	253	# validate section name against the 'submodule "foo"' pattern
249	254	if re.match(r'submodule "(.*)"', section):
250		- path = parser.get(section, 'path')
251		- url = parser.get(section, 'url')
	255	+ yield (parser, section)
252	256
253		- yield (path, url)
	257	+ # List the submodules (path/url tuples) present at the given ref of this repo
	258	+ def submodule_list(self):
	259	+ for parser, section in self._read_gitmodules():
	260	+ path = parser.get(section, 'path')
	261	+ url = parser.get(section, 'url')
	262	+
	263	+ yield (path, url)
254	264
255	265	# Fetch the ref which this mirror requires its submodule to have,
256	266	# at the given ref of this mirror.
257	267	def submodule_ref(self, submodule, ref=None):
258	268	if not ref:
259		- ref = self.ref
	269	+ ref = self.get_ref()
260	270
261	271	# list objects in the parent repo tree to find the commit
262	272	# object that corresponds to the submodule
...	...	@@ -287,6 +297,28 @@ class GitMirror(SourceFetcher):
287	297
288	298	return None
289	299
	300	+ def get_submodule_path(self, url):
	301	+ for parser, section in self._read_gitmodules():
	302	+ parsed_url = parser.get(section, 'url')
	303	+ if parsed_url == url:
	304	+ return parser.get(section, 'path')
	305	+
	306	+ raise SourceError("{}: No submodule found with url '{}'".format(self.source, url))
	307	+
	308	+ def get_path(self):
	309	+ if self._path is None:
	310	+ self._path = self.parent.get_submodule_path()
	311	+
	312	+ return self._path
	313	+
	314	+ def get_ref(self):
	315	+ # The top-level GitMirror may have ref as None, submodules don't.
	316	+ if self._ref is None and self.parent:
	317	+ path = self.get_path()
	318	+ self._ref = self.parent.submodule_ref(path)
	319	+
	320	+ return self._ref
	321	+
290	322
291	323	class GitSource(Source):
292	324	# pylint: disable=attribute-defined-outside-init
...	...	@@ -303,6 +335,8 @@ class GitSource(Source):
303	335	self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
304	336	self.submodules = []
305	337
	338	+ self.using_source_fetchers = (self.original_url != self.translate_url(self.original_url))
	339	+
306	340	# Parse a dict of submodule overrides, stored in the submodule_overrides
307	341	# and submodule_checkout_overrides dictionaries.
308	342	self.submodule_overrides = {}
...	...	@@ -311,6 +345,11 @@ class GitSource(Source):
311	345	for path, _ in self.node_items(modules):
312	346	submodule = self.node_get_member(modules, Mapping, path)
313	347	url = self.node_get_member(submodule, str, 'url', None)
	348	+
	349	+ if self.using_source_fetchers:
	350	+ submodule_mirror = GitMirror(self, None, url, None, parent=self.mirror)
	351	+ self.submodules.append(submodule_mirror)
	352	+
314	353	self.submodule_overrides[path] = url
315	354	if 'checkout' in submodule:
316	355	checkout = self.node_get_member(submodule, bool, 'checkout')
...	...	@@ -326,7 +365,7 @@ class GitSource(Source):
326	365	# Here we want to encode the local name of the repository and
327	366	# the ref, if the user changes the alias to fetch the same sources
328	367	# from another location, it should not effect the cache key.
329		- key = [self.original_url, self.mirror.ref]
	368	+ key = [self.original_url, self.mirror.get_ref()]
330	369
331	370	# Only modify the cache key with checkout_submodules if it's something
332	371	# other than the default behaviour.
...	...	@@ -346,18 +385,18 @@ class GitSource(Source):
346	385	def get_consistency(self):
347	386	if self.have_all_refs():
348	387	return Consistency.CACHED
349		- elif self.mirror.ref is not None:
	388	+ elif self.mirror.get_ref() is not None:
350	389	return Consistency.RESOLVED
351	390	return Consistency.INCONSISTENT
352	391
353	392	def load_ref(self, node):
354		- self.mirror.ref = self.node_get_member(node, str, 'ref', None)
	393	+ self.mirror._ref = self.node_get_member(node, str, 'ref', None)
355	394
356	395	def get_ref(self):
357		- return self.mirror.ref
	396	+ return self.mirror.get_ref()
358	397
359	398	def set_ref(self, ref, node):
360		- node['ref'] = self.mirror.ref = ref
	399	+ node['ref'] = self.mirror._ref = ref
361	400
362	401	def track(self):
363	402
...	...	@@ -376,6 +415,24 @@ class GitSource(Source):
376	415
377	416	return ret
378	417
	418	+ def fetch(self):
	419	+
	420	+ with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True):
	421	+
	422	+ # Here we are only interested in ensuring that our mirror contains
	423	+ # the self.mirror.ref commit.
	424	+ self.mirror.ensure()
	425	+ if not self.mirror.has_ref():
	426	+ self.mirror.fetch()
	427	+
	428	+ self.mirror.assert_ref()
	429	+
	430	+ # Here after performing any fetches, we need to also ensure that
	431	+ # we've cached the desired refs in our mirrors of submodules.
	432	+ #
	433	+ self.refresh_submodules()
	434	+ self.fetch_submodules()
	435	+
379	436	def init_workspace(self, directory):
380	437	# XXX: may wish to refactor this as some code dupe with stage()
381	438	self.refresh_submodules()
...	...	@@ -399,8 +456,9 @@ class GitSource(Source):
399	456	with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
400	457	self.mirror.stage(directory)
401	458	for mirror in self.submodules:
402		- if mirror.path in self.submodule_checkout_overrides:
403		- checkout = self.submodule_checkout_overrides[mirror.path]
	459	+ mirror_path = mirror.get_path()
	460	+ if mirror_path in self.submodule_checkout_overrides:
	461	+ checkout = self.submodule_checkout_overrides[mirror_path]
404	462	else:
405	463	checkout = self.checkout_submodules
406	464
...	...	@@ -408,8 +466,11 @@ class GitSource(Source):
408	466	mirror.stage(directory)
409	467
410	468	def get_source_fetchers(self):
411		- self.refresh_submodules()
412		- return [self.mirror] + self.submodules
	469	+ # If the url does not contain an alias, then it does not need SourceFetchers
	470	+ if self.mirror.url == self.translate_url(self.mirror.url):
	471	+ return []
	472	+ else:
	473	+ return [self.mirror] + self.submodules
413	474
414	475	###########################################################
415	476	# Local Functions #
...	...	@@ -432,6 +493,11 @@ class GitSource(Source):
432	493	# Assumes that we have our mirror and we have the ref which we point to
433	494	#
434	495	def refresh_submodules(self):
	496	+
	497	+ # When using source fetchers, the submodule list is defined by the 'submodules' config field
	498	+ if self.using_source_fetchers:
	499	+ return
	500	+
435	501	self.mirror.ensure()
436	502	submodules = []
437	503
...	...	@@ -454,6 +520,19 @@ class GitSource(Source):
454	520
455	521	self.submodules = submodules
456	522
	523	+ # Ensures that we have mirrored git repositories for all
	524	+ # the submodules existing at the given commit of the main git source.
	525	+ #
	526	+ # Also ensure that these mirrors have the required commits
	527	+ # referred to at the given commit of the main git source.
	528	+ #
	529	+ def fetch_submodules(self):
	530	+ for mirror in self.submodules:
	531	+ mirror.ensure()
	532	+ if not mirror.has_ref():
	533	+ mirror.fetch()
	534	+ mirror.assert_ref()
	535	+
457	536
458	537	# Plugin entry point
459	538	def setup():

buildstream/source.py

---

...	...	@@ -393,8 +393,8 @@ class Source(Plugin):
393	393	"""Get the objects that are used for fetching
394	394
395	395	If this source doesn't download from multiple URLs,
396		- returning None and falling back on the default behaviour
397		- is recommended.
	396	+ returning an empty list and falling back on the default
	397	+ behaviour is recommended.
398	398
399	399	Returns:
400	400	list: A list of SourceFetchers. If SourceFetchers are not supported,

tests/frontend/mirror.py

---

...	...	@@ -139,6 +139,63 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
139	139	result.assert_success()
140	140
141	141
	142	+@pytest.mark.datafiles(DATA_DIR)
	143	+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
	144	+def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
	145	+ bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
	146	+ dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
	147	+ upstream_repodir = os.path.join(str(tmpdir), 'upstream')
	148	+ mirror_repodir = os.path.join(str(tmpdir), 'mirror')
	149	+ project_dir = os.path.join(str(tmpdir), 'project')
	150	+ os.makedirs(project_dir)
	151	+ element_dir = os.path.join(project_dir, 'elements')
	152	+
	153	+ # Create repo objects of the upstream and mirror
	154	+ upstream_repo = create_repo(kind, upstream_repodir)
	155	+ ref = upstream_repo.create(dev_files_path)
	156	+ mirror_repo = upstream_repo.copy(mirror_repodir)
	157	+
	158	+ element = {
	159	+ 'kind': 'import',
	160	+ 'sources': [
	161	+ upstream_repo.source_config(ref=ref)
	162	+ ]
	163	+ }
	164	+
	165	+ element_name = 'test.bst'
	166	+ element_path = os.path.join(element_dir, element_name)
	167	+ full_repo = element['sources'][0]['url']
	168	+ upstream_map, repo_name = os.path.split(full_repo)
	169	+ alias = 'foo-' + kind
	170	+ aliased_repo = alias + ':' + repo_name
	171	+ element['sources'][0]['url'] = aliased_repo
	172	+ full_mirror = mirror_repo.source_config()['url']
	173	+ mirror_map, _ = os.path.split(full_mirror)
	174	+ os.makedirs(element_dir)
	175	+ _yaml.dump(element, element_path)
	176	+
	177	+ project = {
	178	+ 'name': 'test',
	179	+ 'element-path': 'elements',
	180	+ 'aliases': {
	181	+ alias: 'http://www.example.com/'
	182	+ },
	183	+ 'mirrors': [
	184	+ {
	185	+ 'name': 'middle-earth',
	186	+ 'aliases': {
	187	+ alias: [mirror_map + "/"],
	188	+ },
	189	+ },
	190	+ ]
	191	+ }
	192	+ project_file = os.path.join(project_dir, 'project.conf')
	193	+ _yaml.dump(project, project_file)
	194	+
	195	+ result = cli.run(project=project_dir, args=['fetch', element_name])
	196	+ result.assert_success()
	197	+
	198	+
142	199	@pytest.mark.datafiles(DATA_DIR)
143	200	def test_mirror_fetch_multi(cli, tmpdir, datafiles):
144	201	output_file = os.path.join(str(tmpdir), "output.txt")

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.