[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 6 commits: Add --tar option to source-checkout command

[Phil Dawson <gitlab mg gitlab com>]

Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

• f143bfb8
  by Phil Dawson at 2018-11-30T13:26:31Z

  Add --tar option to source-checkout command
  
  This commit is part of the work towards #672
  

• 6d50a233
  by Phil Dawson at 2018-11-30T13:26:31Z

  Add --force / -f option to bst source-checkout command
  
  This commit is part of the work towards #672
  

• 7c3283e5
  by Phil Dawson at 2018-11-30T13:26:32Z

  Add option to source-checkout to generate build scripts
  
  This commit is part of the work towards #672
  

• 41a84a6f
  by Phil Dawson at 2018-11-30T13:26:32Z

  Remove source bundle command
  
  This is part of the work towards #672
  

• da85d613
  by Phil Dawson at 2018-11-30T13:26:32Z

  NEWS: Add entry for new source-checkout command
  

• da5cd243
  by Phil Dawson at 2018-11-30T13:26:32Z

  NEWS: Add entry about the removal of source-bundle command
  

8 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• doc/source/using_commands.rst
• tests/completions/completions.py
• tests/frontend/help.py
• − tests/frontend/source_bundle.py
• tests/frontend/source_checkout.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,17 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o BREAKING CHANGE: The bst source-bundle command has been removed. The
	6	+ functionality it provided has been replaced by the `--include-build-scripts`
	7	+ option of the `bst source-checkout` command. To produce a tarball containing
	8	+ an element's sources and generated build scripts you can do the command
	9	+ `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
	10	+
	11	+ o A `bst source-checkout` command has been added. This command allows an
	12	+ element's sources to be checkout out into a user specified location. For
	13	+ example, `bst source-checkout foo.bst some/path/` will collect the sources
	14	+ of `foo.bst` and place them in the directory `some/path`.
	15	+
5	16	o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
6	17	environment variables. There is already a 'make' element with the same
7	18	variables. Note that this is a breaking change, it will require users to

buildstream/_frontend/cli.py

---

...	...	@@ -668,6 +668,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
668	668	# Source Checkout Command #
669	669	##################################################################
670	670	@cli.command(name='source-checkout', short_help='Checkout sources for an element')
	671	+@click.option('--force', '-f', default=False, is_flag=True,
	672	+ help="Allow files to be overwritten")
671	673	@click.option('--except', 'except_', multiple=True,
672	674	type=click.Path(readable=False),
673	675	help="Except certain dependencies")
...	...	@@ -676,19 +678,26 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
676	678	help='The dependencies whose sources to checkout (default: none)')
677	679	@click.option('--fetch', 'fetch_', default=False, is_flag=True,
678	680	help='Fetch elements if they are not fetched')
679		-@click.argument('element',
680		- type=click.Path(readable=False))
	681	+@click.option('--tar', 'tar', default=False, is_flag=True,
	682	+ help='Create a tarball from the element\'s sources instead of a '
	683	+ 'file tree.')
	684	+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
	685	+@click.argument('element', type=click.Path(readable=False))
681	686	@click.argument('location', type=click.Path())
682	687	@click.pass_obj
683		-def source_checkout(app, element, location, deps, fetch_, except_):
	688	+def source_checkout(app, element, location, force, deps, fetch_, except_,
	689	+ tar, build_scripts):
684	690	"""Checkout sources of an element to the specified location
685	691	"""
686	692	with app.initialized():
687	693	app.stream.source_checkout(element,
688	694	location=location,
	695	+ force=force,
689	696	deps=deps,
690	697	fetch=fetch_,
691		- except_targets=except_)
	698	+ except_targets=except_,
	699	+ tar=tar,
	700	+ include_build_scripts=build_scripts)
692	701
693	702
694	703	##################################################################
...	...	@@ -827,34 +836,3 @@ def workspace_list(app):
827	836
828	837	with app.initialized():
829	838	app.stream.workspace_list()
830		-
831		-
832		-##################################################################
833		-# Source Bundle Command #
834		-##################################################################
835		-@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
836		-@click.option('--except', 'except_', multiple=True,
837		- type=click.Path(readable=False),
838		- help="Elements to except from the tarball")
839		-@click.option('--compression', default='gz',
840		- type=click.Choice(['none', 'gz', 'bz2', 'xz']),
841		- help="Compress the tar file using the given algorithm.")
842		-@click.option('--track', 'track_', default=False, is_flag=True,
843		- help="Track new source references before bundling")
844		-@click.option('--force', '-f', default=False, is_flag=True,
845		- help="Overwrite an existing tarball")
846		-@click.option('--directory', default=os.getcwd(),
847		- help="The directory to write the tarball to")
848		-@click.argument('element',
849		- type=click.Path(readable=False))
850		-@click.pass_obj
851		-def source_bundle(app, element, force, directory,
852		- track_, compression, except_):
853		- """Produce a source bundle to be manually executed
854		- """
855		- with app.initialized():
856		- app.stream.source_bundle(element, directory,
857		- track_first=track_,
858		- force=force,
859		- compression=compression,
860		- except_targets=except_)

buildstream/_stream.py

---

...	...	@@ -436,11 +436,14 @@ class Stream():
436	436	#
437	437	def source_checkout(self, target, *,
438	438	location=None,
	439	+ force=False,
439	440	deps='none',
440	441	fetch=False,
441		- except_targets=()):
	442	+ except_targets=(),
	443	+ tar=False,
	444	+ include_build_scripts=False):
442	445
443		- self._check_location_writable(location)
	446	+ self._check_location_writable(location, force=force, tar=tar)
444	447
445	448	elements, _ = self._load((target,), (),
446	449	selection=deps,
...	...	@@ -454,7 +457,8 @@ class Stream():
454	457
455	458	# Stage all sources determined by scope
456	459	try:
457		- self._write_element_sources(location, elements)
	460	+ self._source_checkout(elements, location, force, deps, fetch,
	461	+ except_targets, tar, include_build_scripts)
458	462	except BstError as e:
459	463	raise StreamError("Error while writing sources"
460	464	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
...	...	@@ -666,87 +670,6 @@ class Stream():
666	670	'workspaces': workspaces
667	671	})
668	672
669		- # source_bundle()
670		- #
671		- # Create a host buildable tarball bundle for the given target.
672		- #
673		- # Args:
674		- # target (str): The target element to bundle
675		- # directory (str): The directory to output the tarball
676		- # track_first (bool): Track new source references before bundling
677		- # compression (str): The compression type to use
678		- # force (bool): Overwrite an existing tarball
679		- #
680		- def source_bundle(self, target, directory, *,
681		- track_first=False,
682		- force=False,
683		- compression="gz",
684		- except_targets=()):
685		-
686		- if track_first:
687		- track_targets = (target,)
688		- else:
689		- track_targets = ()
690		-
691		- elements, track_elements = self._load((target,), track_targets,
692		- selection=PipelineSelection.ALL,
693		- except_targets=except_targets,
694		- track_selection=PipelineSelection.ALL,
695		- fetch_subprojects=True)
696		-
697		- # source-bundle only supports one target
698		- target = self.targets[0]
699		-
700		- self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
701		-
702		- # Find the correct filename for the compression algorithm
703		- tar_location = os.path.join(directory, target.normal_name + ".tar")
704		- if compression != "none":
705		- tar_location += "." + compression
706		-
707		- # Attempt writing a file to generate a good error message
708		- # early
709		- #
710		- # FIXME: A bit hackish
711		- try:
712		- open(tar_location, mode="x")
713		- os.remove(tar_location)
714		- except IOError as e:
715		- raise StreamError("Cannot write to {0}: {1}"
716		- .format(tar_location, e)) from e
717		-
718		- # Fetch and possibly track first
719		- #
720		- self._fetch(elements, track_elements=track_elements)
721		-
722		- # We don't use the scheduler for this as it is almost entirely IO
723		- # bound.
724		-
725		- # Create a temporary directory to build the source tree in
726		- builddir = self._context.builddir
727		- os.makedirs(builddir, exist_ok=True)
728		- prefix = "{}-".format(target.normal_name)
729		-
730		- with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
731		- source_directory = os.path.join(tempdir, 'source')
732		- try:
733		- os.makedirs(source_directory)
734		- except OSError as e:
735		- raise StreamError("Failed to create directory: {}"
736		- .format(e)) from e
737		-
738		- # Any elements that don't implement _write_script
739		- # should not be included in the later stages.
740		- elements = [
741		- element for element in elements
742		- if self._write_element_script(source_directory, element)
743		- ]
744		-
745		- self._write_element_sources(os.path.join(tempdir, "source"), elements)
746		- self._write_build_script(tempdir, elements)
747		- self._collect_sources(tempdir, tar_location,
748		- target.normal_name, compression)
749		-
750	673	# redirect_element_names()
751	674	#
752	675	# Takes a list of element names and returns a list where elements have been
...	...	@@ -1127,6 +1050,51 @@ class Stream():
1127	1050
1128	1051	sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
1129	1052
	1053	+ # Helper function for source_checkout()
	1054	+ def _source_checkout(self, elements,
	1055	+ location=None,
	1056	+ force=False,
	1057	+ deps='none',
	1058	+ fetch=False,
	1059	+ except_targets=(),
	1060	+ tar=False,
	1061	+ include_build_scripts=False):
	1062	+ location = os.path.abspath(location)
	1063	+
	1064	+ # Stage all our sources in a temporary directory. The this
	1065	+ # directory can be used to either construct a tarball or moved
	1066	+ # to the final desired location.
	1067	+ temp_source_dir = self._create_temp_source_dir(elements, include_build_scripts)
	1068	+ if tar:
	1069	+ self._create_tarball(temp_source_dir.name, location)
	1070	+ else:
	1071	+ if force:
	1072	+ try:
	1073	+ shutil.rmtree(location)
	1074	+ except FileNotFoundError:
	1075	+ pass
	1076	+ try:
	1077	+ shutil.move(temp_source_dir.name, location)
	1078	+ except OSError as e:
	1079	+ temp_source_dir.cleanup()
	1080	+ raise StreamError("Failed to checkout sources to {}: {}".format(location, e))
	1081	+
	1082	+ # Ensure the temporary directory is cleaned up. If it has been
	1083	+ # moved temp_source_dir will no longer exist with it's
	1084	+ # original name. This is expected.
	1085	+ try:
	1086	+ temp_source_dir.cleanup()
	1087	+ except FileNotFoundError:
	1088	+ pass
	1089	+
	1090	+ # Construct a TemporaryDirectory containing the sources of elements.
	1091	+ def _create_temp_source_dir(self, elements, include_build_scripts):
	1092	+ tempdir = TemporaryDirectory()
	1093	+ self._write_element_sources(tempdir.name, elements)
	1094	+ if include_build_scripts:
	1095	+ self._write_build_scripts(tempdir.name, elements)
	1096	+ return tempdir
	1097	+
1130	1098	# Write the element build script to the given directory
1131	1099	def _write_element_script(self, directory, element):
1132	1100	try:
...	...	@@ -1143,8 +1111,29 @@ class Stream():
1143	1111	os.makedirs(element_source_dir)
1144	1112	element._stage_sources_at(element_source_dir)
1145	1113
	1114	+ # Create a tarball from the content of directory
	1115	+ def _create_tarball(self, directory, tar_name):
	1116	+ try:
	1117	+ with tarfile.open(name=tar_name, mode='w') as tf:
	1118	+ for item in os.listdir(str(directory)):
	1119	+ file_to_add = os.path.join(directory, item)
	1120	+ tf.add(file_to_add, arcname=item)
	1121	+ except OSError as e:
	1122	+ # If we have a partially constructed tar file, clean up after ourselves
	1123	+ try:
	1124	+ os.remove(tar_name)
	1125	+ except OSError:
	1126	+ pass
	1127	+ raise StreamError("Failed to create tar archive: {}".format(e)) from e
	1128	+
	1129	+ # Write all the build_scripts for elements in the directory location
	1130	+ def _write_build_scripts(self, location, elements):
	1131	+ for element in elements:
	1132	+ self._write_element_script(location, element)
	1133	+ self._write_master_build_script(location, elements)
	1134	+
1146	1135	# Write a master build script to the sandbox
1147		- def _write_build_script(self, directory, elements):
	1136	+ def _write_master_build_script(self, directory, elements):
1148	1137
1149	1138	module_string = ""
1150	1139	for element in elements:

doc/source/using_commands.rst

---

...	...	@@ -86,13 +86,6 @@ project's main directory.
86	86
87	87	----
88	88
89		-.. _invoking_source_bundle:
90		-
91		-.. click:: buildstream._frontend.cli:source_bundle
92		- :prog: bst source bundle
93		-
94		-----
95		-
96	89	.. _invoking_workspace:
97	90
98	91	.. click:: buildstream._frontend.cli:workspace

tests/completions/completions.py

---

...	...	@@ -16,7 +16,6 @@ MAIN_COMMANDS = [
16	16	'shell ',
17	17	'show ',
18	18	'source-checkout ',
19		- 'source-bundle ',
20	19	'track ',
21	20	'workspace '
22	21	]

tests/frontend/help.py

---

...	...	@@ -25,7 +25,6 @@ def test_help_main(cli):
25	25	('push'),
26	26	('shell'),
27	27	('show'),
28		- ('source-bundle'),
29	28	('track'),
30	29	('workspace')
31	30	])

tests/frontend/source_bundle.py deleted

---

1		-#
2		-# Copyright (C) 2018 Bloomberg Finance LP
3		-#
4		-# This program is free software; you can redistribute it and/or
5		-# modify it under the terms of the GNU Lesser General Public
6		-# License as published by the Free Software Foundation; either
7		-# version 2 of the License, or (at your option) any later version.
8		-#
9		-# This library is distributed in the hope that it will be useful,
10		-# but WITHOUT ANY WARRANTY; without even the implied warranty of
11		-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12		-# Lesser General Public License for more details.
13		-#
14		-# You should have received a copy of the GNU Lesser General Public
15		-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
16		-#
17		-# Authors: Chandan Singh <csingh43 bloomberg net>
18		-#
19		-
20		-import os
21		-import tarfile
22		-
23		-import pytest
24		-
25		-from tests.testutils import cli
26		-
27		-# Project directory
28		-DATA_DIR = os.path.join(
29		- os.path.dirname(os.path.realpath(__file__)),
30		- "project",
31		-)
32		-
33		-
34		-@pytest.mark.datafiles(DATA_DIR)
35		-def test_source_bundle(cli, tmpdir, datafiles):
36		- project_path = os.path.join(datafiles.dirname, datafiles.basename)
37		- element_name = 'source-bundle/source-bundle-hello.bst'
38		- normal_name = 'source-bundle-source-bundle-hello'
39		-
40		- # Verify that we can correctly produce a source-bundle
41		- args = ['source-bundle', element_name, '--directory', str(tmpdir)]
42		- result = cli.run(project=project_path, args=args)
43		- result.assert_success()
44		-
45		- # Verify that the source-bundle contains our sources and a build script
46		- with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
47		- assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
48		- assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

tests/frontend/source_checkout.py

---

1	1	import os
2	2	import pytest
	3	+import tarfile
	4	+from pathlib import Path
3	5
4	6	from tests.testutils import cli
5	7
...	...	@@ -39,6 +41,39 @@ def test_source_checkout(datafiles, cli):
39	41	assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
40	42
41	43
	44	+@pytest.mark.datafiles(DATA_DIR)
	45	+@pytest.mark.parametrize('force_flag', ['--force', '-f'])
	46	+def test_source_checkout_force(datafiles, cli, force_flag):
	47	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	48	+ checkout = os.path.join(cli.directory, 'source-checkout')
	49	+ target = 'checkout-deps.bst'
	50	+
	51	+ os.makedirs(os.path.join(checkout, 'some-thing'))
	52	+ # Path(os.path.join(checkout, 'some-file')).touch()
	53	+
	54	+ result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
	55	+ result.assert_success()
	56	+
	57	+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
	58	+
	59	+
	60	+@pytest.mark.datafiles(DATA_DIR)
	61	+def test_source_checkout_tar(datafiles, cli):
	62	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	63	+ checkout = os.path.join(cli.directory, 'source-checkout.tar')
	64	+ target = 'checkout-deps.bst'
	65	+
	66	+ result = cli.run(project=project, args=['source-checkout', '--tar', target, '--deps', 'none', checkout])
	67	+ result.assert_success()
	68	+
	69	+ assert os.path.exists(checkout)
	70	+ with tarfile.open(checkout) as tf:
	71	+ expected_content = os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')
	72	+ tar_members = [f.name for f in tf]
	73	+ for member in tar_members:
	74	+ assert member in expected_content
	75	+
	76	+
42	77	@pytest.mark.datafiles(DATA_DIR)
43	78	@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
44	79	def test_source_checkout_deps(datafiles, cli, deps):
...	...	@@ -119,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
119	154	assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
120	155	else:
121	156	result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
	157	+
	158	+
	159	+@pytest.mark.datafiles(DATA_DIR)
	160	+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
	161	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	162	+ element_name = 'source-bundle/source-bundle-hello.bst'
	163	+ normal_name = 'source-bundle-source-bundle-hello'
	164	+ checkout = os.path.join(str(tmpdir), 'source-checkout')
	165	+
	166	+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
	167	+ result = cli.run(project=project_path, args=args)
	168	+ result.assert_success()
	169	+
	170	+ # There sould be a script for each element (just one in this case) and a top level build script
	171	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	172	+ for script in expected_scripts:
	173	+ assert script in os.listdir(checkout)
	174	+
	175	+
	176	+@pytest.mark.datafiles(DATA_DIR)
	177	+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
	178	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	179	+ element_name = 'source-bundle/source-bundle-hello.bst'
	180	+ normal_name = 'source-bundle-source-bundle-hello'
	181	+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
	182	+
	183	+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
	184	+ result = cli.run(project=project_path, args=args)
	185	+ result.assert_success()
	186	+
	187	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	188	+
	189	+ with tarfile.open(tar_file, 'r') as tf:
	190	+ for script in expected_scripts:
	191	+ assert script in tf.getnames()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.