[Notes] [Git][BuildStream/buildstream][chandan/source-checkout] Add `bst source-checkout` command

[Chandan Singh <gitlab mg gitlab com>]

Title: GitLab

Chandan Singh pushed to branch chandan/source-checkout at BuildStream / buildstream

Commits:

• eaea98a4
  by Chandan Singh at 2018-09-22T18:31:58Z

  Add `bst source-checkout` command
  
  As discussed in
  https://mail.gnome.org/archives/buildstream-list/2018-September/msg00064.html,
  add `bst source-checkout` command. This will allow users to checkout
  sources for a given target.
  

6 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• tests/completions/completions.py
• + tests/frontend/project/elements/checkout-deps.bst
• + tests/frontend/project/files/etc-files/etc/buildstream/config
• + tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -662,6 +662,30 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
662	662	tar=tar)
663	663
664	664
	665	+##################################################################
	666	+# Source Checkout Command #
	667	+##################################################################
	668	+@cli.command(name="source-checkout", short_help="Checkout sources for an element")
	669	+@click.option('--except', 'except_', multiple=True,
	670	+ type=click.Path(readable=False),
	671	+ help="Except certain dependencies")
	672	+@click.option('--deps', '-d', default='none',
	673	+ type=click.Choice(['build', 'none', 'run', 'all']),
	674	+ help='The dependencies whose sources to checkout (default: none)')
	675	+@click.argument('element',
	676	+ type=click.Path(readable=False))
	677	+@click.argument('location', type=click.Path())
	678	+@click.pass_obj
	679	+def source_checkout(app, element, location, deps, except_):
	680	+ """Checkout sources of an element to the specified location
	681	+ """
	682	+ with app.initialized():
	683	+ app.stream.source_checkout(element,
	684	+ location=location,
	685	+ deps=deps,
	686	+ except_targets=except_)
	687	+
	688	+
665	689	##################################################################
666	690	# Workspace Command #
667	691	##################################################################

buildstream/_stream.py

---

...	...	@@ -381,27 +381,7 @@ class Stream():
381	381	elements, _ = self._load((target,), (), fetch_subprojects=True)
382	382	target = elements[0]
383	383
384		- if not tar:
385		- try:
386		- os.makedirs(location, exist_ok=True)
387		- except OSError as e:
388		- raise StreamError("Failed to create checkout directory: '{}'"
389		- .format(e)) from e
390		-
391		- if not tar:
392		- if not os.access(location, os.W_OK):
393		- raise StreamError("Checkout directory '{}' not writable"
394		- .format(location))
395		- if not force and os.listdir(location):
396		- raise StreamError("Checkout directory '{}' not empty"
397		- .format(location))
398		- elif os.path.exists(location) and location != '-':
399		- if not os.access(location, os.W_OK):
400		- raise StreamError("Output file '{}' not writable"
401		- .format(location))
402		- if not force and os.path.exists(location):
403		- raise StreamError("Output file '{}' already exists"
404		- .format(location))
	384	+ self.__check_location_writable(location, force=False, tar=tar)
405	385
406	386	# Stage deps into a temporary sandbox first
407	387	try:
...	...	@@ -438,6 +418,35 @@ class Stream():
438	418	raise StreamError("Error while staging dependencies into a sandbox"
439	419	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
440	420
	421	+ # source_checkout()
	422	+ #
	423	+ # Checkout sources of the target element to the specified location
	424	+ #
	425	+ # Args:
	426	+ # target (str): The target element whose sources to checkout
	427	+ # location (str): Location to checkout the sources to
	428	+ # deps (str): The dependencies to checkout
	429	+ # except_targets (list): List of targets to except from staging
	430	+ #
	431	+ def source_checkout(self, target, *,
	432	+ location=None,
	433	+ deps='none',
	434	+ except_targets=()):
	435	+
	436	+ self.__check_location_writable(location)
	437	+
	438	+ elements, _ = self._load((target,), (),
	439	+ selection=deps,
	440	+ except_targets=except_targets,
	441	+ fetch_subprojects=True)
	442	+
	443	+ # Stage all sources determined by scope
	444	+ try:
	445	+ self._write_element_sources(location, elements)
	446	+ except BstError as e:
	447	+ raise StreamError("Error while staging sources"
	448	+ ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
	449	+
441	450	# workspace_open
442	451	#
443	452	# Open a project workspace
...	...	@@ -721,7 +730,7 @@ class Stream():
721	730	if self._write_element_script(source_directory, element)
722	731	]
723	732
724		- self._write_element_sources(tempdir, elements)
	733	+ self._write_element_sources(os.path.join(tempdir, "source"), elements)
725	734	self._write_build_script(tempdir, elements)
726	735	self._collect_sources(tempdir, tar_location,
727	736	target.normal_name, compression)
...	...	@@ -1084,11 +1093,10 @@ class Stream():
1084	1093	# Write all source elements to the given directory
1085	1094	def _write_element_sources(self, directory, elements):
1086	1095	for element in elements:
1087		- source_dir = os.path.join(directory, "source")
1088		- element_source_dir = os.path.join(source_dir, element.normal_name)
1089		- os.makedirs(element_source_dir)
1090		-
1091		- element._stage_sources_at(element_source_dir)
	1096	+ element_source_dir = os.path.join(directory, element.normal_name)
	1097	+ if list(element.sources()):
	1098	+ os.makedirs(element_source_dir)
	1099	+ element._stage_sources_at(element_source_dir)
1092	1100
1093	1101	# Write a master build script to the sandbox
1094	1102	def _write_build_script(self, directory, elements):
...	...	@@ -1117,3 +1125,29 @@ class Stream():
1117	1125
1118	1126	with tarfile.open(tar_name, permissions) as tar:
1119	1127	tar.add(directory, arcname=element_name)
	1128	+
	1129	+ #############################################################
	1130	+ # Private Methods #
	1131	+ #############################################################
	1132	+
	1133	+ # Check if given location is writable
	1134	+ def __check_location_writable(self, location, force=False, tar=False):
	1135	+ if not tar:
	1136	+ try:
	1137	+ os.makedirs(location, exist_ok=True)
	1138	+ except OSError as e:
	1139	+ raise StreamError("Failed to create checkout directory: '{}'"
	1140	+ .format(e)) from e
	1141	+ if not os.access(location, os.W_OK):
	1142	+ raise StreamError("Checkout directory '{}' not writable"
	1143	+ .format(location))
	1144	+ if not force and os.listdir(location):
	1145	+ raise StreamError("Checkout directory '{}' not empty"
	1146	+ .format(location))
	1147	+ elif os.path.exists(location) and location != '-':
	1148	+ if not os.access(location, os.W_OK):
	1149	+ raise StreamError("Output file '{}' not writable"
	1150	+ .format(location))
	1151	+ if not force and os.path.exists(location):
	1152	+ raise StreamError("Output file '{}' already exists"
	1153	+ .format(location))

tests/completions/completions.py

---

...	...	@@ -15,6 +15,7 @@ MAIN_COMMANDS = [
15	15	'push ',
16	16	'shell ',
17	17	'show ',
	18	+ 'source-checkout',
18	19	'source-bundle ',
19	20	'track ',
20	21	'workspace '

tests/frontend/project/elements/checkout-deps.bst

---

	1	+kind: import
	2	+description: It is important for this element to have both build and runtime dependencies
	3	+sources:
	4	+- kind: local
	5	+ path: files/etc-files
	6	+depends:
	7	+- filename: import-dev.bst
	8	+ type: build
	9	+- filename: import-bin.bst
	10	+ type: runtime

tests/frontend/project/files/etc-files/etc/buildstream/config

---

1

+config

tests/frontend/source_checkout.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from tests.testutils import cli
	5	+
	6	+# Project directory
	7	+DATA_DIR = os.path.join(
	8	+ os.path.dirname(os.path.realpath(__file__)),
	9	+ "project",
	10	+)
	11	+
	12	+
	13	+@pytest.mark.datafiles(DATA_DIR)
	14	+def test_source_checkout(datafiles, cli):
	15	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	16	+ checkout = os.path.join(cli.directory, 'source-checkout')
	17	+ target = 'checkout-deps.bst'
	18	+
	19	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
	20	+ result.assert_success()
	21	+
	22	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	23	+
	24	+
	25	+@pytest.mark.datafiles(DATA_DIR)
	26	+@pytest.mark.parametrize('deps', [("build"), ("none"), ("run"), ("all")])
	27	+def test_source_checkout_deps(datafiles, cli, deps):
	28	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	29	+ checkout = os.path.join(cli.directory, 'source-checkout')
	30	+ target = 'checkout-deps.bst'
	31	+
	32	+ result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
	33	+ result.assert_success()
	34	+
	35	+ # Sources of the target
	36	+ if deps == 'build':
	37	+ assert not os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	38	+ else:
	39	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps'))
	40	+
	41	+ # Sources of the target's build dependencies
	42	+ if deps == 'run':
	43	+ assert not os.path.exists(os.path.join(checkout, 'import-dev'))
	44	+ else:
	45	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
	46	+
	47	+ # Sources of the target's runtime dependencies
	48	+ if deps in ('run', 'all'):
	49	+ assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
	50	+ else:
	51	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	52	+
	53	+
	54	+@pytest.mark.datafiles(DATA_DIR)
	55	+def test_source_checkout_except(datafiles, cli):
	56	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	57	+ checkout = os.path.join(cli.directory, 'source-checkout')
	58	+ target = 'checkout-deps.bst'
	59	+
	60	+ result = cli.run(project=project, args=['source-checkout', target,
	61	+ '--deps', 'all',
	62	+ '--except', 'import-bin.bst',
	63	+ checkout])
	64	+ result.assert_success()
	65	+
	66	+ # Sources for the target should be present
	67	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	68	+
	69	+ # Sources for import-bin.bst should not be present
	70	+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
	71	+
	72	+ # Sources for other dependencies should be present
	73	+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.