[Notes] [Git][BuildStream/buildstream][Qinusty/597-backport-ValueError-fix] 8 commits: element.py: Schedule assemble for key of workspaced elements

[Qinusty <gitlab mg gitlab com>]

Title: GitLab

Qinusty pushed to branch Qinusty/597-backport-ValueError-fix at BuildStream / buildstream

Commits:

• 76a66abf
  by Jürg Billeter at 2018-08-28T13:29:47Z

  element.py: Schedule assemble for key of workspaced elements
  
  For uncached workspaced elements, assemble is required even just to
  calculate the cache key. As dynamic scheduling relies on cache keys,
  schedule assemble for uncached workspace elements even if they have not
  been marked as required yet.
  
  Fixes #461.
  

• 1a96b537
  by Valentin David at 2018-08-28T13:29:48Z

  tests/frontend/workspace.py: Add test for workspaced dependencies
  
  This adds a regression test for #461.
  

• 765dd3d9
  by Jürg Billeter at 2018-08-28T13:29:48Z

  element.py: Do not attempt to pull workspaced elements
  

• 0de2f87b
  by Tristan Van Berkom at 2018-08-28T13:39:18Z

  tests/frontend/workspaces.py: Removing some redundant tests
  
  Removed redundant tests from recently merged !740, this new
  test does not need to run for every different source kind.
  

• 7da1c309
  by Jürg Billeter at 2018-08-28T16:32:34Z

  Merge branch 'juerg/workspaced-dependencies-1.2' into 'bst-1.2'
  
  Fix key invalidation for workspaced dependencies
  
  See merge request BuildStream/buildstream!744

• 291763b0
  by Chandan Singh at 2018-08-29T12:23:21Z

  setup.py: Add more metadata useful for PyPI
  
  - setup.py: List useful links using `project_urls`
  
  - setup.py: Specify minumum python vresion using `python_requires`
  
    `python_requires` is the new way of specifying requirements for python
    version, as documented @
    https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires.
  
    This will also make this requirement appear nicely on the PyPI project
    page.
  
  - setup.py: Add project classifiers
  
    PyPI will use these classifiers to categorize projects while searching
    or browsing. Full list of classifiers can be found at
    https://pypi.org/classifiers.
  

• 557fb81a
  by Tristan Van Berkom at 2018-08-29T12:44:25Z

  Merge branch 'tristan/setup.py-metadata-1.2' into 'bst-1.2'
  
  setup.py: Add more metadata useful for PyPI
  
  See merge request BuildStream/buildstream!749

• 582e0c05
  by Josh Smith at 2018-08-29T13:02:04Z

  Prevent ValueError on URLs without an alias
  
  Fixes #597
  

5 changed files:

• buildstream/element.py
• buildstream/source.py
• buildstream/utils.py
• setup.py
• tests/frontend/workspace.py

Changes:

buildstream/element.py

---

...	...	@@ -1085,9 +1085,12 @@ class Element(Plugin):
1085	1085	# until the full cache query below.
1086	1086	cached = self.__artifacts.contains(self, self.__weak_cache_key)
1087	1087	if (not self.__assemble_scheduled and not self.__assemble_done and
1088		- not cached and not self._pull_pending() and self._is_required()):
1089		- self._schedule_assemble()
1090		- return
	1088	+ not cached and not self._pull_pending()):
	1089	+ # For uncached workspaced elements, assemble is required
	1090	+ # even if we only need the cache key
	1091	+ if self._is_required() or self._get_workspace():
	1092	+ self._schedule_assemble()
	1093	+ return
1091	1094
1092	1095	if self.__strict_cache_key is None:
1093	1096	dependencies = [
...	...	@@ -1107,13 +1110,17 @@ class Element(Plugin):
1107	1110	self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
1108	1111
1109	1112	if (not self.__assemble_scheduled and not self.__assemble_done and
1110		- not self.__cached and not self._pull_pending() and self._is_required()):
	1113	+ not self.__cached and not self._pull_pending()):
1111	1114	# Workspaced sources are considered unstable if a build is pending
1112	1115	# as the build will modify the contents of the workspace.
1113	1116	# Determine as early as possible if a build is pending to discard
1114	1117	# unstable cache keys.
1115		- self._schedule_assemble()
1116		- return
	1118	+
	1119	+ # For uncached workspaced elements, assemble is required
	1120	+ # even if we only need the cache key
	1121	+ if self._is_required() or self._get_workspace():
	1122	+ self._schedule_assemble()
	1123	+ return
1117	1124
1118	1125	if self.__cache_key is None:
1119	1126	# Calculate strong cache key
...	...	@@ -1382,7 +1389,6 @@ class Element(Plugin):
1382	1389	# in a subprocess.
1383	1390	#
1384	1391	def _schedule_assemble(self):
1385		- assert self._is_required()
1386	1392	assert not self.__assemble_scheduled
1387	1393	self.__assemble_scheduled = True
1388	1394
...	...	@@ -1390,6 +1396,8 @@ class Element(Plugin):
1390	1396	for dep in self.dependencies(Scope.BUILD, recurse=False):
1391	1397	dep._set_required()
1392	1398
	1399	+ self._set_required()
	1400	+
1393	1401	# Invalidate workspace key as the build modifies the workspace directory
1394	1402	workspace = self._get_workspace()
1395	1403	if workspace:
...	...	@@ -1579,6 +1587,10 @@ class Element(Plugin):
1579	1587	# (bool): Whether a pull operation is pending
1580	1588	#
1581	1589	def _pull_pending(self):
	1590	+ if self._get_workspace():
	1591	+ # Workspace builds are never pushed to artifact servers
	1592	+ return False
	1593	+
1582	1594	if self.__strong_cached:
1583	1595	# Artifact already in local cache
1584	1596	return False

buildstream/source.py

---

...	...	@@ -184,10 +184,7 @@ class SourceFetcher():
184	184	Args:
185	185	url (str): The url used to download.
186	186	"""
187		- # Not guaranteed to be a valid alias yet.
188		- # Ensuring it's a valid alias currently happens in Project.get_alias_uris
189		- alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
190		- self.__alias = alias
	187	+ self.__alias = _extract_alias(url)
191	188
192	189	#############################################################
193	190	# Private Methods used in BuildStream #
...	...	@@ -386,8 +383,7 @@ class Source(Plugin):
386	383
387	384	*Since: 1.2*
388	385	"""
389		- alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
390		- self.__expected_alias = alias
	386	+ self.__expected_alias = _extract_alias(url)
391	387
392	388	def get_source_fetchers(self):
393	389	"""Get the objects that are used for fetching
...	...	@@ -452,8 +448,7 @@ class Source(Plugin):
452	448	else:
453	449	# Sneakily store the alias if it hasn't already been stored
454	450	if not self.__expected_alias and url and utils._ALIAS_SEPARATOR in url:
455		- url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
456		- self.__expected_alias = url_alias
	451	+ self.mark_download_url(url)
457	452
458	453	project = self._get_project()
459	454	return project.translate_url(url, first_pass=self.__first_pass)
...	...	@@ -804,12 +799,12 @@ class Source(Plugin):
804	799	# Tries to call track for every mirror, stopping once it succeeds
805	800	def __do_track(self):
806	801	project = self._get_project()
807		- # If there are no mirrors, or no aliases to replace, there's nothing to do here.
808	802	alias = self._get_alias()
809	803	if self.__first_pass:
810	804	mirrors = project.first_pass_config.mirrors
811	805	else:
812	806	mirrors = project.config.mirrors
	807	+ # If there are no mirrors, or no aliases to replace, there's nothing to do here.
813	808	if not mirrors or not alias:
814	809	return self.track()
815	810
...	...	@@ -867,3 +862,11 @@ class Source(Plugin):
867	862	_yaml.node_final_assertions(config)
868	863
869	864	return config
	865	+
	866	+
	867	+def _extract_alias(url):
	868	+ parts = url.split(utils._ALIAS_SEPARATOR, 1)
	869	+ if len(parts) > 1 and not parts[0].lower() in utils._URI_SCHEMES:
	870	+ return parts[0]
	871	+ else:
	872	+ return ""

buildstream/utils.py

---

...	...	@@ -44,6 +44,7 @@ from ._exceptions import BstError, ErrorDomain
44	44
45	45	# The separator we use for user specified aliases
46	46	_ALIAS_SEPARATOR = ':'
	47	+_URI_SCHEMES = ["http", "https", "ftp", "file", "git", "sftp", "ssh"]
47	48
48	49
49	50	class UtilError(BstError):

setup.py

---

...	...	@@ -25,7 +25,14 @@ import subprocess
25	25	import sys
26	26	import versioneer
27	27
28		-if sys.version_info[0] != 3 or sys.version_info[1] < 5:
	28	+
	29	+##################################################################
	30	+# Python requirements
	31	+##################################################################
	32	+REQUIRED_PYTHON_MAJOR = 3
	33	+REQUIRED_PYTHON_MINOR = 5
	34	+
	35	+if sys.version_info[0] != REQUIRED_PYTHON_MAJOR or sys.version_info[1] < REQUIRED_PYTHON_MINOR:
29	36	print("BuildStream requires Python >= 3.5")
30	37	sys.exit(1)
31	38
...	...	@@ -242,11 +249,28 @@ setup(name='BuildStream',
242	249
243	250	author='BuildStream Developers',
244	251	author_email='buildstream-list gnome org',
	252	+ classifiers=[
	253	+ 'Environment :: Console',
	254	+ 'Intended Audience :: Developers',
	255	+ 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
	256	+ 'Operating System :: POSIX',
	257	+ 'Programming Language :: Python :: 3',
	258	+ 'Programming Language :: Python :: 3.5',
	259	+ 'Programming Language :: Python :: 3.6',
	260	+ 'Programming Language :: Python :: 3.7',
	261	+ 'Topic :: Software Development :: Build Tools'
	262	+ ],
245	263	description='A framework for modelling build pipelines in YAML',
246	264	license='LGPL',
247	265	long_description=long_description,
248	266	long_description_content_type='text/x-rst; charset=UTF-8',
249	267	url='https://gitlab.com/BuildStream/buildstream',
	268	+ project_urls={
	269	+ 'Documentation': 'https://buildstream.gitlab.io/buildstream/',
	270	+ 'Tracker': 'https://gitlab.com/BuildStream/buildstream/issues',
	271	+ 'Mailing List': 'https://mail.gnome.org/mailman/listinfo/buildstream-list'
	272	+ },
	273	+ python_requires='~={}.{}'.format(REQUIRED_PYTHON_MAJOR, REQUIRED_PYTHON_MINOR),
250	274	packages=find_packages(exclude=('tests', 'tests.*')),
251	275	package_data={'buildstream': ['plugins/*/*.py', 'plugins/*/*.yaml',
252	276	'data/*.yaml', 'data/*.sh.in']},

tests/frontend/workspace.py

---

...	...	@@ -712,3 +712,73 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
712	712	'build', element_name
713	713	])
714	714	result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced")
	715	+
	716	+
	717	+@pytest.mark.datafiles(DATA_DIR)
	718	+@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
	719	+def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
	720	+ checkout = os.path.join(str(tmpdir), 'checkout')
	721	+ element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
	722	+ datafiles, 'git', False)
	723	+
	724	+ element_path = os.path.join(project, 'elements')
	725	+ back_dep_element_name = 'workspace-test-back-dep.bst'
	726	+
	727	+ # Write out our test target
	728	+ element = {
	729	+ 'kind': 'compose',
	730	+ 'depends': [
	731	+ {
	732	+ 'filename': element_name,
	733	+ 'type': 'build'
	734	+ }
	735	+ ]
	736	+ }
	737	+ _yaml.dump(element,
	738	+ os.path.join(element_path,
	739	+ back_dep_element_name))
	740	+
	741	+ # Modify workspace
	742	+ shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
	743	+ os.makedirs(os.path.join(workspace, 'etc'))
	744	+ with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
	745	+ f.write("PONY='pink'")
	746	+
	747	+ # Configure strict mode
	748	+ strict_mode = True
	749	+ if strict != 'strict':
	750	+ strict_mode = False
	751	+ cli.configure({
	752	+ 'projects': {
	753	+ 'test': {
	754	+ 'strict': strict_mode
	755	+ }
	756	+ }
	757	+ })
	758	+
	759	+ # Build artifact with dependency's modified workspace
	760	+ assert cli.get_element_state(project, element_name) == 'buildable'
	761	+ assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
	762	+ assert cli.get_element_state(project, back_dep_element_name) == 'waiting'
	763	+ assert cli.get_element_key(project, back_dep_element_name) == "{:?<64}".format('')
	764	+ result = cli.run(project=project, args=['build', back_dep_element_name])
	765	+ result.assert_success()
	766	+ assert cli.get_element_state(project, element_name) == 'cached'
	767	+ assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
	768	+ assert cli.get_element_state(project, back_dep_element_name) == 'cached'
	769	+ assert cli.get_element_key(project, back_dep_element_name) != "{:?<64}".format('')
	770	+ result = cli.run(project=project, args=['build', back_dep_element_name])
	771	+ result.assert_success()
	772	+
	773	+ # Checkout the result
	774	+ result = cli.run(project=project, args=[
	775	+ 'checkout', back_dep_element_name, checkout
	776	+ ])
	777	+ result.assert_success()
	778	+
	779	+ # Check that the pony.conf from the modified workspace exists
	780	+ filename = os.path.join(checkout, 'etc', 'pony.conf')
	781	+ assert os.path.exists(filename)
	782	+
	783	+ # Check that the original /usr/bin/hello is not in the checkout
	784	+ assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.