[Notes] [Git][BuildStream/buildstream][master] 3 commits: buildstream/_pipeline.py: Simplify error message for assert_consistent()

[Chandan Singh <gitlab mg gitlab com>]

Title: GitLab

Chandan Singh pushed to branch master at BuildStream / buildstream

Commits:

• 30b72244
  by Chandan Singh at 2018-11-19T12:35:01Z

  buildstream/_pipeline.py: Simplify error message for assert_consistent()
  
  Currently, `Pipeline.assert_consistent()` prints an error message like
  the one below when certain elements have inconsistent sources:
  
  ```
  ...
        Element: hello.bst is inconsistent
          Source tar source at hello.bst [line 16 column 2] is missing ref
  ...
  ```
  
  Drop the word "Source" from the beginning of the message as
  "Source <source-kind> source ..." is kind of awkward to read.
  

• 76c5d2f8
  by Chandan Singh at 2018-11-19T12:35:01Z

  Add `bst source-checkout` command
  
  As discussed in
  https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html,
  add `bst source-checkout` command. This will allow users to checkout
  sources for a given target.
  
  * _frontend/cli.py: Add source-checkout command
  * _pipeline.py: Add assert_sources_cached() method
  * _stream.py: Add source_checkout method, abstract out
    __check_location_writable() method that used to part of checkout()
  

• 9f629638
  by Chandan Singh at 2018-11-19T13:03:36Z

  Merge branch 'chandan/source-checkout' into 'master'
  
  Add `bst source-checkout` command
  
  See merge request BuildStream/buildstream!820

7 changed files:

• buildstream/_frontend/cli.py
• buildstream/_pipeline.py
• buildstream/_stream.py
• tests/completions/completions.py
• + tests/frontend/project/elements/checkout-deps.bst
• + tests/frontend/project/files/etc-files/etc/buildstream/config
• + tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -664,6 +664,33 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
664	664	tar=tar)
665	665
666	666
	667	+##################################################################
	668	+# Source Checkout Command #
	669	+##################################################################
	670	+@cli.command(name='source-checkout', short_help='Checkout sources for an element')
	671	+@click.option('--except', 'except_', multiple=True,
	672	+ type=click.Path(readable=False),
	673	+ help="Except certain dependencies")
	674	+@click.option('--deps', '-d', default='none',
	675	+ type=click.Choice(['build', 'none', 'run', 'all']),
	676	+ help='The dependencies whose sources to checkout (default: none)')
	677	+@click.option('--fetch', 'fetch_', default=False, is_flag=True,
	678	+ help='Fetch elements if they are not fetched')
	679	+@click.argument('element',
	680	+ type=click.Path(readable=False))
	681	+@click.argument('location', type=click.Path())
	682	+@click.pass_obj
	683	+def source_checkout(app, element, location, deps, fetch_, except_):
	684	+ """Checkout sources of an element to the specified location
	685	+ """
	686	+ with app.initialized():
	687	+ app.stream.source_checkout(element,
	688	+ location=location,
	689	+ deps=deps,
	690	+ fetch=fetch_,
	691	+ except_targets=except_)
	692	+
	693	+
667	694	##################################################################
668	695	# Workspace Command #
669	696	##################################################################

buildstream/_pipeline.py

---

...	...	@@ -370,7 +370,7 @@ class Pipeline():
370	370	detail += " Element: {} is inconsistent\n".format(element._get_full_name())
371	371	for source in element.sources():
372	372	if source._get_consistency() == Consistency.INCONSISTENT:
373		- detail += " Source {} is missing ref\n".format(source)
	373	+ detail += " {} is missing ref\n".format(source)
374	374	detail += '\n'
375	375	detail += "Try tracking these elements first with `bst track`\n"
376	376
...	...	@@ -383,6 +383,33 @@ class Pipeline():
383	383	detail += " " + element._get_full_name() + "\n"
384	384	raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced")
385	385
	386	+ # assert_sources_cached()
	387	+ #
	388	+ # Asserts that sources for the given list of elements are cached.
	389	+ #
	390	+ # Args:
	391	+ # elements (list): The list of elements
	392	+ #
	393	+ def assert_sources_cached(self, elements):
	394	+ uncached = []
	395	+ with self._context.timed_activity("Checking sources"):
	396	+ for element in elements:
	397	+ if element._get_consistency() != Consistency.CACHED:
	398	+ uncached.append(element)
	399	+
	400	+ if uncached:
	401	+ detail = "Sources are not cached for the following elements:\n\n"
	402	+ for element in uncached:
	403	+ detail += " Following sources for element: {} are not cached:\n".format(element._get_full_name())
	404	+ for source in element.sources():
	405	+ if source._get_consistency() != Consistency.CACHED:
	406	+ detail += " {}\n".format(source)
	407	+ detail += '\n'
	408	+ detail += "Try fetching these elements first with `bst fetch`,\n" + \
	409	+ "or run this command with `--fetch` option\n"
	410	+
	411	+ raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
	412	+
386	413	#############################################################
387	414	# Private Methods #
388	415	#############################################################

buildstream/_stream.py

---

...	...	@@ -379,27 +379,7 @@ class Stream():
379	379	elements, _ = self._load((target,), (), fetch_subprojects=True)
380	380	target = elements[0]
381	381
382		- if not tar:
383		- try:
384		- os.makedirs(location, exist_ok=True)
385		- except OSError as e:
386		- raise StreamError("Failed to create checkout directory: '{}'"
387		- .format(e)) from e
388		-
389		- if not tar:
390		- if not os.access(location, os.W_OK):
391		- raise StreamError("Checkout directory '{}' not writable"
392		- .format(location))
393		- if not force and os.listdir(location):
394		- raise StreamError("Checkout directory '{}' not empty"
395		- .format(location))
396		- elif os.path.exists(location) and location != '-':
397		- if not os.access(location, os.W_OK):
398		- raise StreamError("Output file '{}' not writable"
399		- .format(location))
400		- if not force and os.path.exists(location):
401		- raise StreamError("Output file '{}' already exists"
402		- .format(location))
	382	+ self._check_location_writable(location, force=force, tar=tar)
403	383
404	384	# Stage deps into a temporary sandbox first
405	385	try:
...	...	@@ -443,6 +423,42 @@ class Stream():
443	423	raise StreamError("Error while staging dependencies into a sandbox"
444	424	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
445	425
	426	+ # source_checkout()
	427	+ #
	428	+ # Checkout sources of the target element to the specified location
	429	+ #
	430	+ # Args:
	431	+ # target (str): The target element whose sources to checkout
	432	+ # location (str): Location to checkout the sources to
	433	+ # deps (str): The dependencies to checkout
	434	+ # fetch (bool): Whether to fetch missing sources
	435	+ # except_targets (list): List of targets to except from staging
	436	+ #
	437	+ def source_checkout(self, target, *,
	438	+ location=None,
	439	+ deps='none',
	440	+ fetch=False,
	441	+ except_targets=()):
	442	+
	443	+ self._check_location_writable(location)
	444	+
	445	+ elements, _ = self._load((target,), (),
	446	+ selection=deps,
	447	+ except_targets=except_targets,
	448	+ fetch_subprojects=True)
	449	+
	450	+ # Assert all sources are cached
	451	+ if fetch:
	452	+ self._fetch(elements)
	453	+ self._pipeline.assert_sources_cached(elements)
	454	+
	455	+ # Stage all sources determined by scope
	456	+ try:
	457	+ self._write_element_sources(location, elements)
	458	+ except BstError as e:
	459	+ raise StreamError("Error while writing sources"
	460	+ ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
	461	+
446	462	# workspace_open
447	463	#
448	464	# Open a project workspace
...	...	@@ -726,7 +742,7 @@ class Stream():
726	742	if self._write_element_script(source_directory, element)
727	743	]
728	744
729		- self._write_element_sources(tempdir, elements)
	745	+ self._write_element_sources(os.path.join(tempdir, "source"), elements)
730	746	self._write_build_script(tempdir, elements)
731	747	self._collect_sources(tempdir, tar_location,
732	748	target.normal_name, compression)
...	...	@@ -1068,6 +1084,39 @@ class Stream():
1068	1084	self._enqueue_plan(fetch_plan)
1069	1085	self._run()
1070	1086
	1087	+ # _check_location_writable()
	1088	+ #
	1089	+ # Check if given location is writable.
	1090	+ #
	1091	+ # Args:
	1092	+ # location (str): Destination path
	1093	+ # force (bool): Allow files to be overwritten
	1094	+ # tar (bool): Whether destination is a tarball
	1095	+ #
	1096	+ # Raises:
	1097	+ # (StreamError): If the destination is not writable
	1098	+ #
	1099	+ def _check_location_writable(self, location, force=False, tar=False):
	1100	+ if not tar:
	1101	+ try:
	1102	+ os.makedirs(location, exist_ok=True)
	1103	+ except OSError as e:
	1104	+ raise StreamError("Failed to create destination directory: '{}'"
	1105	+ .format(e)) from e
	1106	+ if not os.access(location, os.W_OK):
	1107	+ raise StreamError("Destination directory '{}' not writable"
	1108	+ .format(location))
	1109	+ if not force and os.listdir(location):
	1110	+ raise StreamError("Destination directory '{}' not empty"
	1111	+ .format(location))
	1112	+ elif os.path.exists(location) and location != '-':
	1113	+ if not os.access(location, os.W_OK):
	1114	+ raise StreamError("Output file '{}' not writable"
	1115	+ .format(location))
	1116	+ if not force and os.path.exists(location):
	1117	+ raise StreamError("Output file '{}' already exists"
	1118	+ .format(location))
	1119	+
1071	1120	# Helper function for checkout()
1072	1121	#
1073	1122	def _checkout_hardlinks(self, sandbox_vroot, directory):
...	...	@@ -1089,11 +1138,10 @@ class Stream():
1089	1138	# Write all source elements to the given directory
1090	1139	def _write_element_sources(self, directory, elements):
1091	1140	for element in elements:
1092		- source_dir = os.path.join(directory, "source")
1093		- element_source_dir = os.path.join(source_dir, element.normal_name)
1094		- os.makedirs(element_source_dir)
1095		-
1096		- element._stage_sources_at(element_source_dir)
	1141	+ element_source_dir = self._get_element_dirname(directory, element)
	1142	+ if list(element.sources()):
	1143	+ os.makedirs(element_source_dir)
	1144	+ element._stage_sources_at(element_source_dir)
1097	1145
1098	1146	# Write a master build script to the sandbox
1099	1147	def _write_build_script(self, directory, elements):
...	...	@@ -1122,3 +1170,25 @@ class Stream():
1122	1170
1123	1171	with tarfile.open(tar_name, permissions) as tar:
1124	1172	tar.add(directory, arcname=element_name)
	1173	+
	1174	+ # _get_element_dirname()
	1175	+ #
	1176	+ # Get path to directory for an element based on its normal name.
	1177	+ #
	1178	+ # For cross-junction elements, the path will be prefixed with the name
	1179	+ # of the junction element.
	1180	+ #
	1181	+ # Args:
	1182	+ # directory (str): path to base directory
	1183	+ # element (Element): the element
	1184	+ #
	1185	+ # Returns:
	1186	+ # (str): Path to directory for this element
	1187	+ #
	1188	+ def _get_element_dirname(self, directory, element):
	1189	+ parts = [element.normal_name]
	1190	+ while element._get_project() != self._project:
	1191	+ element = element._get_project().junction
	1192	+ parts.append(element.normal_name)
	1193	+
	1194	+ return os.path.join(directory, *reversed(parts))

tests/completions/completions.py

---

...	...	@@ -15,6 +15,7 @@ MAIN_COMMANDS = [
15	15	'push ',
16	16	'shell ',
17	17	'show ',
	18	+ 'source-checkout ',
18	19	'source-bundle ',
19	20	'track ',
20	21	'workspace '

tests/frontend/project/elements/checkout-deps.bst

---

	1	+kind: import
	2	+description: It is important for this element to have both build and runtime dependencies
	3	+sources:
	4	+- kind: local
	5	+ path: files/etc-files
	6	+depends:
	7	+- filename: import-dev.bst
	8	+ type: build
	9	+- filename: import-bin.bst
	10	+ type: runtime

tests/frontend/project/files/etc-files/etc/buildstream/config

---

1

+config

tests/frontend/source_checkout.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from tests.testutils import cli
	5	+
	6	+from buildstream import utils, _yaml
	7	+from buildstream._exceptions import ErrorDomain, LoadErrorReason
	8	+
	9	+# Project directory
	10	+DATA_DIR = os.path.join(
	11	+ os.path.dirname(os.path.realpath(__file__)),
	12	+ 'project',
	13	+)
	14	+
	15	+
	16	+def generate_remote_import_element(input_path, output_path):
	17	+ return {
	18	+ 'kind': 'import',
	19	+ 'sources': [
	20	+ {
	21	+ 'kind': 'remote',
	22	+ 'url': 'file://{}'.format(input_path),
	23	+ 'filename': output_path,
	24	+ 'ref': utils.sha256sum(input_path),
	25	+ }
	26	+ ]
	27	+ }
	28	+
	29	+
	30	+@pytest.mark.datafiles(DATA_DIR)
	31	+def test_source_checkout(datafiles, cli):
	32	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	33	+ checkout = os.path.join(cli.directory, 'source-checkout')
	34	+ target = 'checkout-deps.bst'
	35	+
	36	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
	37	+ result.assert_success()
	38	+
	39	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	40	+
	41	+
	42	+@pytest.mark.datafiles(DATA_DIR)
	43	+@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
	44	+def test_source_checkout_deps(datafiles, cli, deps):
	45	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	46	+ checkout = os.path.join(cli.directory, 'source-checkout')
	47	+ target = 'checkout-deps.bst'
	48	+
	49	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
	50	+ result.assert_success()
	51	+
	52	+ # Sources of the target
	53	+ if deps == 'build':
	54	+ assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
	55	+ else:
	56	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	57	+
	58	+ # Sources of the target's build dependencies
	59	+ if deps in ('build', 'all'):
	60	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
	61	+ else:
	62	+ assert not os.path.exists(os.path.join(checkout, 'import-dev'))
	63	+
	64	+ # Sources of the target's runtime dependencies
	65	+ if deps in ('run', 'all'):
	66	+ assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
	67	+ else:
	68	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	69	+
	70	+
	71	+@pytest.mark.datafiles(DATA_DIR)
	72	+def test_source_checkout_except(datafiles, cli):
	73	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	74	+ checkout = os.path.join(cli.directory, 'source-checkout')
	75	+ target = 'checkout-deps.bst'
	76	+
	77	+ result = cli.run(project=project, args=['source-checkout', target,
	78	+ '--deps', 'all',
	79	+ '--except', 'import-bin.bst',
	80	+ checkout])
	81	+ result.assert_success()
	82	+
	83	+ # Sources for the target should be present
	84	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	85	+
	86	+ # Sources for import-bin.bst should not be present
	87	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	88	+
	89	+ # Sources for other dependencies should be present
	90	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
	91	+
	92	+
	93	+@pytest.mark.datafiles(DATA_DIR)
	94	+@pytest.mark.parametrize('fetch', [(False), (True)])
	95	+def test_source_checkout_fetch(datafiles, cli, fetch):
	96	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	97	+ checkout = os.path.join(cli.directory, 'source-checkout')
	98	+ target = 'remote-import-dev.bst'
	99	+ target_path = os.path.join(project, 'elements', target)
	100	+
	101	+ # Create an element with remote source
	102	+ element = generate_remote_import_element(
	103	+ os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
	104	+ 'pony.h')
	105	+ _yaml.dump(element, target_path)
	106	+
	107	+ # Testing --fetch option requires that we do not have the sources
	108	+ # cached already
	109	+ assert cli.get_element_state(project, target) == 'fetch needed'
	110	+
	111	+ args = ['source-checkout']
	112	+ if fetch:
	113	+ args += ['--fetch']
	114	+ args += [target, checkout]
	115	+ result = cli.run(project=project, args=args)
	116	+
	117	+ if fetch:
	118	+ result.assert_success()
	119	+ assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
	120	+ else:
	121	+ result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.