[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 4 commits: Add option to source-checkout to generate build scripts

[Phil Dawson <gitlab mg gitlab com>]

Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

• 24dbbb08
  by Phil Dawson at 2018-12-04T11:39:28Z

  Add option to source-checkout to generate build scripts
  
  This commit is part of the work towards #672
  

• e0aea036
  by Phil Dawson at 2018-12-04T11:39:28Z

  Remove source bundle command
  
  This is part of the work towards #672
  

• aa125577
  by Phil Dawson at 2018-12-04T11:39:28Z

  NEWS: Add entry for new source-checkout command
  

• 72828f7c
  by Phil Dawson at 2018-12-04T11:39:28Z

  NEWS: Add entry about the removal of source-bundle command
  

8 changed files:

• NEWS
• buildstream/_frontend/cli.py
• buildstream/_stream.py
• doc/source/using_commands.rst
• tests/completions/completions.py
• tests/frontend/help.py
• − tests/frontend/source_bundle.py
• tests/frontend/source_checkout.py

Changes:

NEWS

---

...	...	@@ -2,6 +2,17 @@
2	2	buildstream 1.3.1
3	3	=================
4	4
	5	+ o BREAKING CHANGE: The bst source-bundle command has been removed. The
	6	+ functionality it provided has been replaced by the `--include-build-scripts`
	7	+ option of the `bst source-checkout` command. To produce a tarball containing
	8	+ an element's sources and generated build scripts you can do the command
	9	+ `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
	10	+
	11	+ o A `bst source-checkout` command has been added. This command allows an
	12	+ element's sources to be checkout out into a user specified location. For
	13	+ example, `bst source-checkout foo.bst some/path/` will collect the sources
	14	+ of `foo.bst` and place them in the directory `some/path`.
	15	+
5	16	o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
6	17	environment variables. There is already a 'make' element with the same
7	18	variables. Note that this is a breaking change, it will require users to

buildstream/_frontend/cli.py

---

...	...	@@ -681,11 +681,12 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
681	681	@click.option('--tar', 'tar', default=False, is_flag=True,
682	682	help='Create a tarball from the element\'s sources instead of a '
683	683	'file tree.')
684		-@click.argument('element',
685		- type=click.Path(readable=False))
	684	+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
	685	+@click.argument('element', type=click.Path(readable=False))
686	686	@click.argument('location', type=click.Path())
687	687	@click.pass_obj
688		-def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
	688	+def source_checkout(app, element, location, force, deps, fetch_, except_,
	689	+ tar, build_scripts):
689	690	"""Checkout sources of an element to the specified location
690	691	"""
691	692	with app.initialized():
...	...	@@ -695,7 +696,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
695	696	deps=deps,
696	697	fetch=fetch_,
697	698	except_targets=except_,
698		- tar=tar)
	699	+ tar=tar,
	700	+ include_build_scripts=build_scripts)
699	701
700	702
701	703	##################################################################
...	...	@@ -834,34 +836,3 @@ def workspace_list(app):
834	836
835	837	with app.initialized():
836	838	app.stream.workspace_list()
837		-
838		-
839		-##################################################################
840		-# Source Bundle Command #
841		-##################################################################
842		-@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
843		-@click.option('--except', 'except_', multiple=True,
844		- type=click.Path(readable=False),
845		- help="Elements to except from the tarball")
846		-@click.option('--compression', default='gz',
847		- type=click.Choice(['none', 'gz', 'bz2', 'xz']),
848		- help="Compress the tar file using the given algorithm.")
849		-@click.option('--track', 'track_', default=False, is_flag=True,
850		- help="Track new source references before bundling")
851		-@click.option('--force', '-f', default=False, is_flag=True,
852		- help="Overwrite an existing tarball")
853		-@click.option('--directory', default=os.getcwd(),
854		- help="The directory to write the tarball to")
855		-@click.argument('element',
856		- type=click.Path(readable=False))
857		-@click.pass_obj
858		-def source_bundle(app, element, force, directory,
859		- track_, compression, except_):
860		- """Produce a source bundle to be manually executed
861		- """
862		- with app.initialized():
863		- app.stream.source_bundle(element, directory,
864		- track_first=track_,
865		- force=force,
866		- compression=compression,
867		- except_targets=except_)

buildstream/_stream.py

---

...	...	@@ -440,7 +440,8 @@ class Stream():
440	440	deps='none',
441	441	fetch=False,
442	442	except_targets=(),
443		- tar=False):
	443	+ tar=False,
	444	+ include_build_scripts=False):
444	445
445	446	self._check_location_writable(location, force=force, tar=tar)
446	447
...	...	@@ -456,10 +457,8 @@ class Stream():
456	457
457	458	# Stage all sources determined by scope
458	459	try:
459		- if tar:
460		- self._create_source_tarball(location, elements)
461		- else:
462		- self._write_element_sources(location, elements)
	460	+ self._source_checkout(elements, location, force, deps, fetch,
	461	+ except_targets, tar, include_build_scripts)
463	462	except BstError as e:
464	463	raise StreamError("Error while writing sources"
465	464	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
...	...	@@ -671,87 +670,6 @@ class Stream():
671	670	'workspaces': workspaces
672	671	})
673	672
674		- # source_bundle()
675		- #
676		- # Create a host buildable tarball bundle for the given target.
677		- #
678		- # Args:
679		- # target (str): The target element to bundle
680		- # directory (str): The directory to output the tarball
681		- # track_first (bool): Track new source references before bundling
682		- # compression (str): The compression type to use
683		- # force (bool): Overwrite an existing tarball
684		- #
685		- def source_bundle(self, target, directory, *,
686		- track_first=False,
687		- force=False,
688		- compression="gz",
689		- except_targets=()):
690		-
691		- if track_first:
692		- track_targets = (target,)
693		- else:
694		- track_targets = ()
695		-
696		- elements, track_elements = self._load((target,), track_targets,
697		- selection=PipelineSelection.ALL,
698		- except_targets=except_targets,
699		- track_selection=PipelineSelection.ALL,
700		- fetch_subprojects=True)
701		-
702		- # source-bundle only supports one target
703		- target = self.targets[0]
704		-
705		- self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
706		-
707		- # Find the correct filename for the compression algorithm
708		- tar_location = os.path.join(directory, target.normal_name + ".tar")
709		- if compression != "none":
710		- tar_location += "." + compression
711		-
712		- # Attempt writing a file to generate a good error message
713		- # early
714		- #
715		- # FIXME: A bit hackish
716		- try:
717		- open(tar_location, mode="x")
718		- os.remove(tar_location)
719		- except IOError as e:
720		- raise StreamError("Cannot write to {0}: {1}"
721		- .format(tar_location, e)) from e
722		-
723		- # Fetch and possibly track first
724		- #
725		- self._fetch(elements, track_elements=track_elements)
726		-
727		- # We don't use the scheduler for this as it is almost entirely IO
728		- # bound.
729		-
730		- # Create a temporary directory to build the source tree in
731		- builddir = self._context.builddir
732		- os.makedirs(builddir, exist_ok=True)
733		- prefix = "{}-".format(target.normal_name)
734		-
735		- with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
736		- source_directory = os.path.join(tempdir, 'source')
737		- try:
738		- os.makedirs(source_directory)
739		- except OSError as e:
740		- raise StreamError("Failed to create directory: {}"
741		- .format(e)) from e
742		-
743		- # Any elements that don't implement _write_script
744		- # should not be included in the later stages.
745		- elements = [
746		- element for element in elements
747		- if self._write_element_script(source_directory, element)
748		- ]
749		-
750		- self._write_element_sources(os.path.join(tempdir, "source"), elements)
751		- self._write_build_script(tempdir, elements)
752		- self._collect_sources(tempdir, tar_location,
753		- target.normal_name, compression)
754		-
755	673	# redirect_element_names()
756	674	#
757	675	# Takes a list of element names and returns a list where elements have been
...	...	@@ -1135,6 +1053,7 @@ class Stream():
1135	1053	# Helper function for source_checkout()
1136	1054	def _source_checkout(self, elements,
1137	1055	location=None,
	1056	+ force=False,
1138	1057	deps='none',
1139	1058	fetch=False,
1140	1059	except_targets=(),
...	...	@@ -1205,17 +1124,33 @@ class Stream():
1205	1124	os.makedirs(element_source_dir)
1206	1125	element._stage_sources_at(element_source_dir)
1207	1126
1208		- # Create a tarball containing the sources of each element in elements
1209		- def _create_source_tarball(self, directory, elements):
1210		- with tarfile.open(name=directory, mode='w') as tf:
1211		- with TemporaryDirectory() as tmpdir:
1212		- self._write_element_sources(tmpdir, elements)
1213		- for item in os.listdir(tmpdir):
1214		- file_to_add = os.path.join(tmpdir, item)
1215		- tf.add(file_to_add, arcname=item)
	1127	+ # Create a tarball from the content of directory
	1128	+ def _create_tarball(self, directory, tar_name):
	1129	+ try:
	1130	+ with utils.save_file_atomic(tar_name, mode='wb') as f:
	1131	+ # This TarFile does not need to be explicitly closed
	1132	+ # as the underlying file object will be closed be the
	1133	+ # save_file_atomic contect manager
	1134	+ tarball = tarfile.open(fileobj=f, mode='w')
	1135	+ for item in os.listdir(str(directory)):
	1136	+ file_to_add = os.path.join(directory, item)
	1137	+ tarball.add(file_to_add, arcname=item)
	1138	+ except OSError as e:
	1139	+ # If we have a partially constructed tar file, clean up after ourselves
	1140	+ try:
	1141	+ os.remove(tar_name)
	1142	+ except OSError:
	1143	+ pass
	1144	+ raise StreamError("Failed to create tar archive: {}".format(e)) from e
	1145	+
	1146	+ # Write all the build_scripts for elements in the directory location
	1147	+ def _write_build_scripts(self, location, elements):
	1148	+ for element in elements:
	1149	+ self._write_element_script(location, element)
	1150	+ self._write_master_build_script(location, elements)
1216	1151
1217	1152	# Write a master build script to the sandbox
1218		- def _write_build_script(self, directory, elements):
	1153	+ def _write_master_build_script(self, directory, elements):
1219	1154
1220	1155	module_string = ""
1221	1156	for element in elements:

doc/source/using_commands.rst

---

...	...	@@ -86,13 +86,6 @@ project's main directory.
86	86
87	87	----
88	88
89		-.. _invoking_source_bundle:
90		-
91		-.. click:: buildstream._frontend.cli:source_bundle
92		- :prog: bst source bundle
93		-
94		-----
95		-
96	89	.. _invoking_workspace:
97	90
98	91	.. click:: buildstream._frontend.cli:workspace

tests/completions/completions.py

---

...	...	@@ -16,7 +16,6 @@ MAIN_COMMANDS = [
16	16	'shell ',
17	17	'show ',
18	18	'source-checkout ',
19		- 'source-bundle ',
20	19	'track ',
21	20	'workspace '
22	21	]

tests/frontend/help.py

---

...	...	@@ -25,7 +25,6 @@ def test_help_main(cli):
25	25	('push'),
26	26	('shell'),
27	27	('show'),
28		- ('source-bundle'),
29	28	('track'),
30	29	('workspace')
31	30	])

tests/frontend/source_bundle.py deleted

---

1		-#
2		-# Copyright (C) 2018 Bloomberg Finance LP
3		-#
4		-# This program is free software; you can redistribute it and/or
5		-# modify it under the terms of the GNU Lesser General Public
6		-# License as published by the Free Software Foundation; either
7		-# version 2 of the License, or (at your option) any later version.
8		-#
9		-# This library is distributed in the hope that it will be useful,
10		-# but WITHOUT ANY WARRANTY; without even the implied warranty of
11		-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12		-# Lesser General Public License for more details.
13		-#
14		-# You should have received a copy of the GNU Lesser General Public
15		-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
16		-#
17		-# Authors: Chandan Singh <csingh43 bloomberg net>
18		-#
19		-
20		-import os
21		-import tarfile
22		-
23		-import pytest
24		-
25		-from tests.testutils import cli
26		-
27		-# Project directory
28		-DATA_DIR = os.path.join(
29		- os.path.dirname(os.path.realpath(__file__)),
30		- "project",
31		-)
32		-
33		-
34		-@pytest.mark.datafiles(DATA_DIR)
35		-def test_source_bundle(cli, tmpdir, datafiles):
36		- project_path = os.path.join(datafiles.dirname, datafiles.basename)
37		- element_name = 'source-bundle/source-bundle-hello.bst'
38		- normal_name = 'source-bundle-source-bundle-hello'
39		-
40		- # Verify that we can correctly produce a source-bundle
41		- args = ['source-bundle', element_name, '--directory', str(tmpdir)]
42		- result = cli.run(project=project_path, args=args)
43		- result.assert_success()
44		-
45		- # Verify that the source-bundle contains our sources and a build script
46		- with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
47		- assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
48		- assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

tests/frontend/source_checkout.py

---

...	...	@@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
154	154	assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
155	155	else:
156	156	result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
	157	+
	158	+
	159	+@pytest.mark.datafiles(DATA_DIR)
	160	+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
	161	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	162	+ element_name = 'source-bundle/source-bundle-hello.bst'
	163	+ normal_name = 'source-bundle-source-bundle-hello'
	164	+ checkout = os.path.join(str(tmpdir), 'source-checkout')
	165	+
	166	+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
	167	+ result = cli.run(project=project_path, args=args)
	168	+ result.assert_success()
	169	+
	170	+ # There sould be a script for each element (just one in this case) and a top level build script
	171	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	172	+ for script in expected_scripts:
	173	+ assert script in os.listdir(checkout)
	174	+
	175	+
	176	+@pytest.mark.datafiles(DATA_DIR)
	177	+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
	178	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	179	+ element_name = 'source-bundle/source-bundle-hello.bst'
	180	+ normal_name = 'source-bundle-source-bundle-hello'
	181	+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
	182	+
	183	+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
	184	+ result = cli.run(project=project_path, args=args)
	185	+ result.assert_success()
	186	+
	187	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	188	+
	189	+ with tarfile.open(tar_file, 'r') as tf:
	190	+ for script in expected_scripts:
	191	+ assert script in tf.getnames()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.