[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 3 commits: Add --fetch option to source-checkout command

[Phil Dawson <gitlab mg gitlab com>]

Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

• c310bb68
  by Phil Dawson at 2018-11-05T11:21:22Z

  Add --fetch option to source-checkout command
  

• ada6ba24
  by Phil Dawson at 2018-11-05T11:21:23Z

  Add --force / -f option to bst source-checkout command
  

• 830d03a4
  by Phil Dawson at 2018-11-16T17:39:54Z

  Add option to source-checkout to generate build scripts
  

4 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• tests/frontend/source_bundle.py
• tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -666,6 +666,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
666	666	# Source Checkout Command #
667	667	##################################################################
668	668	@cli.command(name='source-checkout', short_help='Checkout sources for an element')
	669	+@click.option('--force', '-f', default=False, is_flag=True,
	670	+ help="Allow files to be overwritten")
669	671	@click.option('--except', 'except_', multiple=True,
670	672	type=click.Path(readable=False),
671	673	help="Except certain dependencies")
...	...	@@ -674,19 +676,28 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
674	676	help='The dependencies whose sources to checkout (default: none)')
675	677	@click.option('--fetch', 'fetch_', default=False, is_flag=True,
676	678	help='Fetch elements if they are not fetched')
	679	+@click.option('--tar', 'tar_', default=False, is_flag=True,
	680	+ help='Create a tarball from the element\'s sources instead of a '
	681	+ 'file tree. If LOCATION is \'-\', the tarball will be dumped '
	682	+ 'to the standard output.')
	683	+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
677	684	@click.argument('element',
678	685	type=click.Path(readable=False))
679	686	@click.argument('location', type=click.Path())
680	687	@click.pass_obj
681		-def source_checkout(app, element, location, deps, fetch_, except_):
	688	+def source_checkout(app, element, location, force, deps, fetch_, except_,
	689	+ tar_, build_scripts):
682	690	"""Checkout sources of an element to the specified location
683	691	"""
684	692	with app.initialized():
685	693	app.stream.source_checkout(element,
686	694	location=location,
	695	+ force=force,
687	696	deps=deps,
688	697	fetch=fetch_,
689		- except_targets=except_)
	698	+ except_targets=except_,
	699	+ tar=tar_,
	700	+ include_build_scripts=build_scripts)
690	701
691	702
692	703	##################################################################

buildstream/_stream.py

---

...	...	@@ -429,11 +429,14 @@ class Stream():
429	429	#
430	430	def source_checkout(self, target, *,
431	431	location=None,
	432	+ force=False,
432	433	deps='none',
433	434	fetch=False,
434		- except_targets=()):
	435	+ except_targets=(),
	436	+ tar=False,
	437	+ include_build_scripts=False):
435	438
436		- self._check_location_writable(location)
	439	+ self._check_location_writable(location, force=force, tar=tar)
437	440
438	441	elements, _ = self._load((target,), (),
439	442	selection=deps,
...	...	@@ -447,11 +450,33 @@ class Stream():
447	450
448	451	# Stage all sources determined by scope
449	452	try:
450		- self._write_element_sources(location, elements)
	453	+ self._source_checkout(elements, location, force, deps, fetch,
	454	+ except_targets, tar, include_build_scripts)
451	455	except BstError as e:
452	456	raise StreamError("Error while writing sources"
453	457	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
454	458
	459	+ def _source_checkout(self, elements,
	460	+ location=None,
	461	+ force=False,
	462	+ deps='none',
	463	+ fetch=False,
	464	+ except_targets=(),
	465	+ tar=False,
	466	+ include_build_scripts=False):
	467	+ location = os.path.abspath(location)
	468	+ if tar:
	469	+ self._create_source_tarball(location, elements, include_build_scripts)
	470	+ else:
	471	+ self._write_element_sources(location, elements)
	472	+ if include_build_scripts:
	473	+ self._write_build_scripts(location, elements)
	474	+
	475	+ def _write_build_scripts(self, location, elements):
	476	+ for element in elements:
	477	+ self._write_element_script(location, element)
	478	+ self._write_master_build_script(location, elements)
	479	+
455	480	# workspace_open
456	481	#
457	482	# Open a project workspace
...	...	@@ -736,7 +761,7 @@ class Stream():
736	761	]
737	762
738	763	self._write_element_sources(os.path.join(tempdir, "source"), elements)
739		- self._write_build_script(tempdir, elements)
	764	+ self._write_master_build_script(tempdir, elements)
740	765	self._collect_sources(tempdir, tar_location,
741	766	target.normal_name, compression)
742	767
...	...	@@ -1136,8 +1161,30 @@ class Stream():
1136	1161	os.makedirs(element_source_dir)
1137	1162	element._stage_sources_at(element_source_dir)
1138	1163
	1164	+ # Create a tarball containing the sources of each element in elements
	1165	+ def _create_source_tarball(self, tar_name, elements, include_build_scripts):
	1166	+ # Stage sources into a temporary directory then create a tarball from this
	1167	+ with TemporaryDirectory() as tmpdir:
	1168	+ self._write_element_sources(tmpdir, elements)
	1169	+ if include_build_scripts:
	1170	+ self._write_build_scripts(tmpdir, elements)
	1171	+ self._create_tar_from_directory(tar_name, tmpdir)
	1172	+
	1173	+ # Create a tarball from the content of directory
	1174	+ def _create_tar_from_directory(self, tar_name, directory):
	1175	+ try:
	1176	+ with tarfile.open(name=tar_name, mode='w') as tf:
	1177	+ for item in os.listdir(str(directory)):
	1178	+ file_to_add = os.path.join(directory, item)
	1179	+ tf.add(file_to_add, arcname=item)
	1180	+ except OSError as e:
	1181	+ # If we have a partially constructed tar file, clean up after ourselves
	1182	+ if os.path.exists(tar_name):
	1183	+ os.remove(tar_name)
	1184	+ raise StreamError("Failed to create tar archieve: {}".format(e)) from e
	1185	+
1139	1186	# Write a master build script to the sandbox
1140		- def _write_build_script(self, directory, elements):
	1187	+ def _write_master_build_script(self, directory, elements):
1141	1188
1142	1189	module_string = ""
1143	1190	for element in elements:

tests/frontend/source_bundle.py

---

...	...	@@ -46,3 +46,4 @@ def test_source_bundle(cli, tmpdir, datafiles):
46	46	with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
47	47	assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
48	48	assert os.path.join(normal_name, 'build.sh') in bundle.getnames()
	49	+

tests/frontend/source_checkout.py

---

1	1	import os
2	2	import pytest
	3	+import tarfile
	4	+from pathlib import Path
3	5
4	6	from tests.testutils import cli
5	7
...	...	@@ -39,6 +41,39 @@ def test_source_checkout(datafiles, cli):
39	41	assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
40	42
41	43
	44	+@pytest.mark.datafiles(DATA_DIR)
	45	+@pytest.mark.parametrize('force_flag', ['--force', '-f'])
	46	+def test_source_checkout_force(datafiles, cli, force_flag):
	47	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	48	+ checkout = os.path.join(cli.directory, 'source-checkout')
	49	+ target = 'checkout-deps.bst'
	50	+
	51	+ os.makedirs(os.path.join(checkout, 'some-thing'))
	52	+ # Path(os.path.join(checkout, 'some-file')).touch()
	53	+
	54	+ result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
	55	+ result.assert_success()
	56	+
	57	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	58	+
	59	+
	60	+@pytest.mark.datafiles(DATA_DIR)
	61	+def test_source_checkout_tar(datafiles, cli):
	62	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	63	+ checkout = os.path.join(cli.directory, 'source-checkout.tar')
	64	+ target = 'checkout-deps.bst'
	65	+
	66	+ result = cli.run(project=project, args=['source-checkout', '--tar', target, '--deps', 'none', checkout])
	67	+ result.assert_success()
	68	+
	69	+ assert os.path.exists(checkout)
	70	+ with tarfile.open(checkout) as tf:
	71	+ expected_content = os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')
	72	+ tar_members = [f.name for f in tf]
	73	+ for member in tar_members:
	74	+ assert member in expected_content
	75	+
	76	+
42	77	@pytest.mark.datafiles(DATA_DIR)
43	78	@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
44	79	def test_source_checkout_deps(datafiles, cli, deps):
...	...	@@ -119,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
119	154	assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
120	155	else:
121	156	result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
	157	+
	158	+
	159	+@pytest.mark.datafiles(DATA_DIR)
	160	+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
	161	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	162	+ element_name = 'source-bundle/source-bundle-hello.bst'
	163	+ normal_name = 'source-bundle-source-bundle-hello'
	164	+ checkout = os.path.join(str(tmpdir), 'source-checkout')
	165	+
	166	+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
	167	+ result = cli.run(project=project_path, args=args)
	168	+ result.assert_success()
	169	+
	170	+ # There sould be a script for each element (just one in this case) and a top level build script
	171	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	172	+ for script in expected_scripts:
	173	+ assert script in os.listdir(checkout)
	174	+
	175	+
	176	+@pytest.mark.datafiles(DATA_DIR)
	177	+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
	178	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	179	+ element_name = 'source-bundle/source-bundle-hello.bst'
	180	+ normal_name = 'source-bundle-source-bundle-hello'
	181	+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
	182	+
	183	+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
	184	+ result = cli.run(project=project_path, args=args)
	185	+ result.assert_success()
	186	+
	187	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	188	+
	189	+ with tarfile.open(tar_file, 'r') as tf:
	190	+ for script in expected_scripts:
	191	+ assert script in tf.getnames()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.