[Notes] [Git][BuildStream/buildstream][537-mirror-fallback-does-not-work-for-git] 6 commits: .gitlab-ci.yml: Remove unused script "install.sh"

[Jonathan Maw <gitlab mg gitlab com>]

Title: GitLab

Jonathan Maw pushed to branch 537-mirror-fallback-does-not-work-for-git at BuildStream / buildstream

Commits:

• ed653fbc
  by Chandan Singh at 2018-08-08T00:06:28Z

  .gitlab-ci.yml: Remove unused script "install.sh"
  

• 65f382f1
  by Chandan Singh at 2018-08-08T10:52:32Z

  Merge branch 'chandan/remove-dead-code' into 'master'
  
  .gitlab-ci.yml: Remove unused script "install.sh"
  
  See merge request BuildStream/buildstream!617

• b22ce3cb
  by Jonathan Maw at 2018-08-08T15:50:41Z

  tests: Test that fetching passes when upstream is absent
  

• f969d810
  by Jonathan Maw at 2018-08-08T15:50:41Z

  source.py: Fix documentation on get_source_fetchers
  

• c3befa7b
  by Jonathan Maw at 2018-08-08T15:50:41Z

  git.py: Provide a list of source fetchers without downloading anything
  

• a80bd2dc
  by Jonathan Maw at 2018-08-08T15:50:41Z

  Exclude ostree from tests on fetching with missing upstream
  
  OSTree sources currently fail and there is an issue to fix it, but this
  branch can't be merged when tests fail.
  

4 changed files:

• .gitlab-ci.yml
• buildstream/plugins/sources/git.py
• buildstream/source.py
• tests/frontend/mirror.py

Changes:

.gitlab-ci.yml

---

...	...	@@ -26,15 +26,6 @@ source_dist:
26	26	- tar -ztf dist/*
27	27	- tarball=$(cd dist && echo $(ls *))
28	28
29		- # Create an installer script
30		- - |
31		- cat > dist/install.sh << EOF
32		- #!/bin/sh
33		- tar -zxf ${tarball}
34		- cd ${tarball%.tar.gz}
35		- pip3 install --no-index .
36		- EOF
37		-
38	29	# unpack tarball as `dist/buildstream` directory
39	30	- |
40	31	cat > dist/unpack.sh << EOF
...	...	@@ -44,7 +35,6 @@ source_dist:
44	35	EOF
45	36
46	37	# Make our helpers executable
47		- - chmod +x dist/install.sh
48	38	- chmod +x dist/unpack.sh
49	39	artifacts:
50	40	paths:

buildstream/plugins/sources/git.py

---

...	...	@@ -91,16 +91,18 @@ GIT_MODULES = '.gitmodules'
91	91	#
92	92	class GitMirror(SourceFetcher):
93	93
94		- def __init__(self, source, path, url, ref):
	94	+ def __init__(self, source, path, url, ref, *, parent=None):
95	95
96	96	super().__init__()
97	97	self.source = source
98		- self.path = path
	98	+ self.parent = parent
99	99	self.url = url
100		- self.ref = ref
101	100	self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
102	101	self.mark_download_url(url)
103	102
	103	+ self._path = path
	104	+ self._ref = ref
	105	+
104	106	# Ensures that the mirror exists
105	107	def ensure(self, alias_override=None):
106	108
...	...	@@ -223,8 +225,7 @@ class GitMirror(SourceFetcher):
223	225	fail="Failed to checkout git ref {}".format(self.ref),
224	226	cwd=fullpath)
225	227
226		- # List the submodules (path/url tuples) present at the given ref of this repo
227		- def submodule_list(self):
	228	+ def _read_gitmodules(self):
228	229	modules = "{}:{}".format(self.ref, GIT_MODULES)
229	230	exit_code, output = self.source.check_output(
230	231	[self.source.host_git, 'show', modules], cwd=self.mirror)
...	...	@@ -247,10 +248,15 @@ class GitMirror(SourceFetcher):
247	248	for section in parser.sections():
248	249	# validate section name against the 'submodule "foo"' pattern
249	250	if re.match(r'submodule "(.*)"', section):
250		- path = parser.get(section, 'path')
251		- url = parser.get(section, 'url')
	251	+ yield (parser, section)
252	252
253		- yield (path, url)
	253	+ # List the submodules (path/url tuples) present at the given ref of this repo
	254	+ def submodule_list(self):
	255	+ for parser, section in self._read_gitmodules():
	256	+ path = parser.get(section, 'path')
	257	+ url = parser.get(section, 'url')
	258	+
	259	+ yield (path, url)
254	260
255	261	# Fetch the ref which this mirror requires its submodule to have,
256	262	# at the given ref of this mirror.
...	...	@@ -287,6 +293,76 @@ class GitMirror(SourceFetcher):
287	293
288	294	return None
289	295
	296	+ def get_submodule_path(self, url):
	297	+ for parser, section in self._read_gitmodules():
	298	+ parsed_url = parser.get(section, 'url')
	299	+ if parsed_url == url:
	300	+ return parser.get(section, 'path')
	301	+
	302	+ raise SourceError("{}: No submodule found with url '{}'".format(self.source, url))
	303	+
	304	+ @property
	305	+ def path(self):
	306	+ if self._path is None:
	307	+ self._path = self.parent.get_submodule_path(self.url)
	308	+
	309	+ return self._path
	310	+
	311	+ @property
	312	+ def ref(self):
	313	+ # The top-level GitMirror may have ref as None, submodules don't.
	314	+ if self._ref is None and self.parent:
	315	+ self._ref = self.parent.submodule_ref(self.path)
	316	+
	317	+ return self._ref
	318	+
	319	+ @ref.setter
	320	+ def ref(self, ref):
	321	+ self._ref = ref
	322	+
	323	+
	324	+# A SourceFetcher that may also check for, and have submodules of its own.
	325	+class TopLevelGitMirror(GitMirror):
	326	+ def __init__(self, source, path, url, ref):
	327	+ super().__init__(source, path, url, ref)
	328	+ self.auto_submodules = []
	329	+
	330	+ def fetch(self, alias_override=None):
	331	+ super().fetch(alias_override)
	332	+ self.refresh_submodules()
	333	+
	334	+ # auto_submodules do not have aliases, so don't need an override
	335	+ for mirror in self.auto_submodules:
	336	+ mirror.fetch()
	337	+
	338	+ # Refreshes the GitMirror objects for submodules
	339	+ #
	340	+ # Assumes that we have our mirror and we have the ref which we point to
	341	+ #
	342	+ def refresh_submodules(self):
	343	+ self.ensure()
	344	+
	345	+ excluded_paths = list([s.path for s in self.source.manual_submodules])
	346	+ submodules = []
	347	+
	348	+ # XXX Here we should issue a warning if either:
	349	+ # A.) A submodule exists but is not defined in the element configuration
	350	+ # B.) The element configuration configures submodules which dont exist at the current ref
	351	+ #
	352	+ for path, url in self.submodule_list():
	353	+ if path in excluded_paths:
	354	+ continue
	355	+ else:
	356	+ self.source.warn("Unexpected submodule detected with path '{}' and url '{}'"
	357	+ .format(path, url))
	358	+
	359	+ ref = self.submodule_ref(path)
	360	+ if ref is not None:
	361	+ mirror = GitMirror(self, path, url, ref)
	362	+ submodules.append(mirror)
	363	+
	364	+ self.auto_submodules = submodules
	365	+
290	366
291	367	class GitSource(Source):
292	368	# pylint: disable=attribute-defined-outside-init
...	...	@@ -298,10 +374,10 @@ class GitSource(Source):
298	374	self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
299	375
300	376	self.original_url = self.node_get_member(node, str, 'url')
301		- self.mirror = GitMirror(self, '', self.original_url, ref)
	377	+ self.mirror = TopLevelGitMirror(self, '', self.original_url, ref)
302	378	self.tracking = self.node_get_member(node, str, 'track', None)
303	379	self.checkout_submodules = self.node_get_member(node, bool, 'checkout-submodules', True)
304		- self.submodules = []
	380	+ self.manual_submodules = []
305	381
306	382	# Parse a dict of submodule overrides, stored in the submodule_overrides
307	383	# and submodule_checkout_overrides dictionaries.
...	...	@@ -311,6 +387,9 @@ class GitSource(Source):
311	387	for path, _ in self.node_items(modules):
312	388	submodule = self.node_get_member(modules, Mapping, path)
313	389	url = self.node_get_member(submodule, str, 'url', None)
	390	+ submodule_mirror = GitMirror(self, None, url, None, parent=self.mirror)
	391	+ self.manual_submodules.append(submodule_mirror)
	392	+
314	393	self.submodule_overrides[path] = url
315	394	if 'checkout' in submodule:
316	395	checkout = self.node_get_member(submodule, bool, 'checkout')
...	...	@@ -384,7 +463,7 @@ class GitSource(Source):
384	463
385	464	def init_workspace(self, directory):
386	465	# XXX: may wish to refactor this as some code dupe with stage()
387		- self.refresh_submodules()
	466	+ self.mirror.refresh_submodules()
388	467
389	468	with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
390	469	self.mirror.init_workspace(directory)
...	...	@@ -398,7 +477,7 @@ class GitSource(Source):
398	477	# with submodules present (source needed fetching) and
399	478	# we may not know about the submodule yet come time to build.
400	479	#
401		- self.refresh_submodules()
	480	+ self.mirror.refresh_submodules()
402	481
403	482	# Stage the main repo in the specified directory
404	483	#
...	...	@@ -414,17 +493,20 @@ class GitSource(Source):
414	493	mirror.stage(directory)
415	494
416	495	def get_source_fetchers(self):
417		- self.refresh_submodules()
418		- return [self.mirror] + self.submodules
	496	+ return [self.mirror] + self.manual_submodules
419	497
420	498	###########################################################
421	499	# Local Functions #
422	500	###########################################################
	501	+ @property
	502	+ def submodules(self):
	503	+ return self.manual_submodules + self.mirror.auto_submodules
	504	+
423	505	def have_all_refs(self):
424	506	if not self.mirror.has_ref():
425	507	return False
426	508
427		- self.refresh_submodules()
	509	+ self.mirror.refresh_submodules()
428	510	for mirror in self.submodules:
429	511	if not os.path.exists(mirror.mirror):
430	512	return False
...	...	@@ -433,33 +515,6 @@ class GitSource(Source):
433	515
434	516	return True
435	517
436		- # Refreshes the GitMirror objects for submodules
437		- #
438		- # Assumes that we have our mirror and we have the ref which we point to
439		- #
440		- def refresh_submodules(self):
441		- self.mirror.ensure()
442		- submodules = []
443		-
444		- # XXX Here we should issue a warning if either:
445		- # A.) A submodule exists but is not defined in the element configuration
446		- # B.) The element configuration configures submodules which dont exist at the current ref
447		- #
448		- for path, url in self.mirror.submodule_list():
449		-
450		- # Allow configuration to override the upstream
451		- # location of the submodules.
452		- override_url = self.submodule_overrides.get(path)
453		- if override_url:
454		- url = override_url
455		-
456		- ref = self.mirror.submodule_ref(path)
457		- if ref is not None:
458		- mirror = GitMirror(self, path, url, ref)
459		- submodules.append(mirror)
460		-
461		- self.submodules = submodules
462		-
463	518
464	519	# Plugin entry point
465	520	def setup():

buildstream/source.py

---

...	...	@@ -393,8 +393,8 @@ class Source(Plugin):
393	393	"""Get the objects that are used for fetching
394	394
395	395	If this source doesn't download from multiple URLs,
396		- returning None and falling back on the default behaviour
397		- is recommended.
	396	+ returning an empty list and falling back on the default
	397	+ behaviour is recommended.
398	398
399	399	Returns:
400	400	list: A list of SourceFetchers. If SourceFetchers are not supported,

tests/frontend/mirror.py

---

...	...	@@ -139,6 +139,63 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
139	139	result.assert_success()
140	140
141	141
	142	+@pytest.mark.datafiles(DATA_DIR)
	143	+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS if kind is not "ostree"])
	144	+def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
	145	+ bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
	146	+ dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
	147	+ upstream_repodir = os.path.join(str(tmpdir), 'upstream')
	148	+ mirror_repodir = os.path.join(str(tmpdir), 'mirror')
	149	+ project_dir = os.path.join(str(tmpdir), 'project')
	150	+ os.makedirs(project_dir)
	151	+ element_dir = os.path.join(project_dir, 'elements')
	152	+
	153	+ # Create repo objects of the upstream and mirror
	154	+ upstream_repo = create_repo(kind, upstream_repodir)
	155	+ ref = upstream_repo.create(dev_files_path)
	156	+ mirror_repo = upstream_repo.copy(mirror_repodir)
	157	+
	158	+ element = {
	159	+ 'kind': 'import',
	160	+ 'sources': [
	161	+ upstream_repo.source_config(ref=ref)
	162	+ ]
	163	+ }
	164	+
	165	+ element_name = 'test.bst'
	166	+ element_path = os.path.join(element_dir, element_name)
	167	+ full_repo = element['sources'][0]['url']
	168	+ upstream_map, repo_name = os.path.split(full_repo)
	169	+ alias = 'foo-' + kind
	170	+ aliased_repo = alias + ':' + repo_name
	171	+ element['sources'][0]['url'] = aliased_repo
	172	+ full_mirror = mirror_repo.source_config()['url']
	173	+ mirror_map, _ = os.path.split(full_mirror)
	174	+ os.makedirs(element_dir)
	175	+ _yaml.dump(element, element_path)
	176	+
	177	+ project = {
	178	+ 'name': 'test',
	179	+ 'element-path': 'elements',
	180	+ 'aliases': {
	181	+ alias: 'http://www.example.com/'
	182	+ },
	183	+ 'mirrors': [
	184	+ {
	185	+ 'name': 'middle-earth',
	186	+ 'aliases': {
	187	+ alias: [mirror_map + "/"],
	188	+ },
	189	+ },
	190	+ ]
	191	+ }
	192	+ project_file = os.path.join(project_dir, 'project.conf')
	193	+ _yaml.dump(project, project_file)
	194	+
	195	+ result = cli.run(project=project_dir, args=['fetch', element_name])
	196	+ result.assert_success()
	197	+
	198	+
142	199	@pytest.mark.datafiles(DATA_DIR)
143	200	def test_mirror_fetch_multi(cli, tmpdir, datafiles):
144	201	output_file = os.path.join(str(tmpdir), "output.txt")

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.