[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 2 commits: Add option to source-checkout to generate build scripts

[Phil Dawson <gitlab mg gitlab com>]

Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

• ec83dd8a
  by Phil Dawson at 2018-11-19T10:51:31Z

  Add option to source-checkout to generate build scripts
  

• 54a9ce54
  by Phil Dawson at 2018-11-19T10:51:31Z

  Remove source bundle command
  
  This is part of the work towards #672
  

5 changed files:

• buildstream/_frontend/cli.py
• buildstream/_stream.py
• tests/completions/completions.py
• − tests/frontend/source_bundle.py
• tests/frontend/source_checkout.py

Changes:

buildstream/_frontend/cli.py

---

...	...	@@ -682,11 +682,13 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
682	682	help='Create a tarball from the element\'s sources instead of a '
683	683	'file tree. If LOCATION is \'-\', the tarball will be dumped '
684	684	'to the standard output.')
	685	+@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
685	686	@click.argument('element',
686	687	type=click.Path(readable=False))
687	688	@click.argument('location', type=click.Path())
688	689	@click.pass_obj
689		-def source_checkout(app, element, location, force, deps, fetch_, except_, tar_):
	690	+def source_checkout(app, element, location, force, deps, fetch_, except_,
	691	+ tar_, build_scripts):
690	692	"""Checkout sources of an element to the specified location
691	693	"""
692	694	with app.initialized():
...	...	@@ -696,7 +698,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar_):
696	698	deps=deps,
697	699	fetch=fetch_,
698	700	except_targets=except_,
699		- tar=tar_)
	701	+ tar=tar_,
	702	+ include_build_scripts=build_scripts)
700	703
701	704
702	705	##################################################################
...	...	@@ -835,34 +838,3 @@ def workspace_list(app):
835	838
836	839	with app.initialized():
837	840	app.stream.workspace_list()
838		-
839		-
840		-##################################################################
841		-# Source Bundle Command #
842		-##################################################################
843		-@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
844		-@click.option('--except', 'except_', multiple=True,
845		- type=click.Path(readable=False),
846		- help="Elements to except from the tarball")
847		-@click.option('--compression', default='gz',
848		- type=click.Choice(['none', 'gz', 'bz2', 'xz']),
849		- help="Compress the tar file using the given algorithm.")
850		-@click.option('--track', 'track_', default=False, is_flag=True,
851		- help="Track new source references before bundling")
852		-@click.option('--force', '-f', default=False, is_flag=True,
853		- help="Overwrite an existing tarball")
854		-@click.option('--directory', default=os.getcwd(),
855		- help="The directory to write the tarball to")
856		-@click.argument('element',
857		- type=click.Path(readable=False))
858		-@click.pass_obj
859		-def source_bundle(app, element, force, directory,
860		- track_, compression, except_):
861		- """Produce a source bundle to be manually executed
862		- """
863		- with app.initialized():
864		- app.stream.source_bundle(element, directory,
865		- track_first=track_,
866		- force=force,
867		- compression=compression,
868		- except_targets=except_)

buildstream/_stream.py

---

...	...	@@ -440,7 +440,8 @@ class Stream():
440	440	deps='none',
441	441	fetch=False,
442	442	except_targets=(),
443		- tar=False):
	443	+ tar=False,
	444	+ include_build_scripts=False):
444	445
445	446	self._check_location_writable(location, force=force, tar=tar)
446	447
...	...	@@ -456,10 +457,8 @@ class Stream():
456	457
457	458	# Stage all sources determined by scope
458	459	try:
459		- if tar:
460		- self._create_source_tarball(location, elements)
461		- else:
462		- self._write_element_sources(location, elements)
	460	+ self._source_checkout(elements, location, force, deps, fetch,
	461	+ except_targets, tar, include_build_scripts)
463	462	except BstError as e:
464	463	raise StreamError("Error while writing sources"
465	464	": '{}'".format(e), detail=e.detail, reason=e.reason) from e
...	...	@@ -671,87 +670,6 @@ class Stream():
671	670	'workspaces': workspaces
672	671	})
673	672
674		- # source_bundle()
675		- #
676		- # Create a host buildable tarball bundle for the given target.
677		- #
678		- # Args:
679		- # target (str): The target element to bundle
680		- # directory (str): The directory to output the tarball
681		- # track_first (bool): Track new source references before bundling
682		- # compression (str): The compression type to use
683		- # force (bool): Overwrite an existing tarball
684		- #
685		- def source_bundle(self, target, directory, *,
686		- track_first=False,
687		- force=False,
688		- compression="gz",
689		- except_targets=()):
690		-
691		- if track_first:
692		- track_targets = (target,)
693		- else:
694		- track_targets = ()
695		-
696		- elements, track_elements = self._load((target,), track_targets,
697		- selection=PipelineSelection.ALL,
698		- except_targets=except_targets,
699		- track_selection=PipelineSelection.ALL,
700		- fetch_subprojects=True)
701		-
702		- # source-bundle only supports one target
703		- target = self.targets[0]
704		-
705		- self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
706		-
707		- # Find the correct filename for the compression algorithm
708		- tar_location = os.path.join(directory, target.normal_name + ".tar")
709		- if compression != "none":
710		- tar_location += "." + compression
711		-
712		- # Attempt writing a file to generate a good error message
713		- # early
714		- #
715		- # FIXME: A bit hackish
716		- try:
717		- open(tar_location, mode="x")
718		- os.remove(tar_location)
719		- except IOError as e:
720		- raise StreamError("Cannot write to {0}: {1}"
721		- .format(tar_location, e)) from e
722		-
723		- # Fetch and possibly track first
724		- #
725		- self._fetch(elements, track_elements=track_elements)
726		-
727		- # We don't use the scheduler for this as it is almost entirely IO
728		- # bound.
729		-
730		- # Create a temporary directory to build the source tree in
731		- builddir = self._context.builddir
732		- os.makedirs(builddir, exist_ok=True)
733		- prefix = "{}-".format(target.normal_name)
734		-
735		- with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
736		- source_directory = os.path.join(tempdir, 'source')
737		- try:
738		- os.makedirs(source_directory)
739		- except OSError as e:
740		- raise StreamError("Failed to create directory: {}"
741		- .format(e)) from e
742		-
743		- # Any elements that don't implement _write_script
744		- # should not be included in the later stages.
745		- elements = [
746		- element for element in elements
747		- if self._write_element_script(source_directory, element)
748		- ]
749		-
750		- self._write_element_sources(os.path.join(tempdir, "source"), elements)
751		- self._write_build_script(tempdir, elements)
752		- self._collect_sources(tempdir, tar_location,
753		- target.normal_name, compression)
754		-
755	673	# redirect_element_names()
756	674	#
757	675	# Takes a list of element names and returns a list where elements have been
...	...	@@ -1132,6 +1050,23 @@ class Stream():
1132	1050
1133	1051	sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
1134	1052
	1053	+ # Helper function for source_checkout()
	1054	+ def _source_checkout(self, elements,
	1055	+ location=None,
	1056	+ force=False,
	1057	+ deps='none',
	1058	+ fetch=False,
	1059	+ except_targets=(),
	1060	+ tar=False,
	1061	+ include_build_scripts=False):
	1062	+ location = os.path.abspath(location)
	1063	+ if tar:
	1064	+ self._create_source_tarball(location, elements, include_build_scripts)
	1065	+ else:
	1066	+ self._write_element_sources(location, elements)
	1067	+ if include_build_scripts:
	1068	+ self._write_build_scripts(location, elements)
	1069	+
1135	1070	# Write the element build script to the given directory
1136	1071	def _write_element_script(self, directory, element):
1137	1072	try:
...	...	@@ -1149,16 +1084,35 @@ class Stream():
1149	1084	element._stage_sources_at(element_source_dir)
1150	1085
1151	1086	# Create a tarball containing the sources of each element in elements
1152		- def _create_source_tarball(self, directory, elements):
1153		- with tarfile.open(name=directory, mode='w') as tf:
1154		- with TemporaryDirectory() as tmpdir:
1155		- self._write_element_sources(tmpdir, elements)
1156		- for item in os.listdir(tmpdir):
1157		- file_to_add = os.path.join(tmpdir, item)
	1087	+ def _create_source_tarball(self, tar_name, elements, include_build_scripts):
	1088	+ # Stage sources into a temporary directory then create a tarball from this
	1089	+ with TemporaryDirectory() as tmpdir:
	1090	+ self._write_element_sources(tmpdir, elements)
	1091	+ if include_build_scripts:
	1092	+ self._write_build_scripts(tmpdir, elements)
	1093	+ self._create_tar_from_directory(tar_name, tmpdir)
	1094	+
	1095	+ # Create a tarball from the content of directory
	1096	+ def _create_tar_from_directory(self, tar_name, directory):
	1097	+ try:
	1098	+ with tarfile.open(name=tar_name, mode='w') as tf:
	1099	+ for item in os.listdir(str(directory)):
	1100	+ file_to_add = os.path.join(directory, item)
1158	1101	tf.add(file_to_add, arcname=item)
	1102	+ except OSError as e:
	1103	+ # If we have a partially constructed tar file, clean up after ourselves
	1104	+ if os.path.exists(tar_name):
	1105	+ os.remove(tar_name)
	1106	+ raise StreamError("Failed to create tar archieve: {}".format(e)) from e
	1107	+
	1108	+ # Write all the build_scripts for elements in the directory location
	1109	+ def _write_build_scripts(self, location, elements):
	1110	+ for element in elements:
	1111	+ self._write_element_script(location, element)
	1112	+ self._write_master_build_script(location, elements)
1159	1113
1160	1114	# Write a master build script to the sandbox
1161		- def _write_build_script(self, directory, elements):
	1115	+ def _write_master_build_script(self, directory, elements):
1162	1116
1163	1117	module_string = ""
1164	1118	for element in elements:

tests/completions/completions.py

---

...	...	@@ -16,7 +16,6 @@ MAIN_COMMANDS = [
16	16	'shell ',
17	17	'show ',
18	18	'source-checkout ',
19		- 'source-bundle ',
20	19	'track ',
21	20	'workspace '
22	21	]

tests/frontend/source_bundle.py deleted

---

1		-#
2		-# Copyright (C) 2018 Bloomberg Finance LP
3		-#
4		-# This program is free software; you can redistribute it and/or
5		-# modify it under the terms of the GNU Lesser General Public
6		-# License as published by the Free Software Foundation; either
7		-# version 2 of the License, or (at your option) any later version.
8		-#
9		-# This library is distributed in the hope that it will be useful,
10		-# but WITHOUT ANY WARRANTY; without even the implied warranty of
11		-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12		-# Lesser General Public License for more details.
13		-#
14		-# You should have received a copy of the GNU Lesser General Public
15		-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
16		-#
17		-# Authors: Chandan Singh <csingh43 bloomberg net>
18		-#
19		-
20		-import os
21		-import tarfile
22		-
23		-import pytest
24		-
25		-from tests.testutils import cli
26		-
27		-# Project directory
28		-DATA_DIR = os.path.join(
29		- os.path.dirname(os.path.realpath(__file__)),
30		- "project",
31		-)
32		-
33		-
34		-@pytest.mark.datafiles(DATA_DIR)
35		-def test_source_bundle(cli, tmpdir, datafiles):
36		- project_path = os.path.join(datafiles.dirname, datafiles.basename)
37		- element_name = 'source-bundle/source-bundle-hello.bst'
38		- normal_name = 'source-bundle-source-bundle-hello'
39		-
40		- # Verify that we can correctly produce a source-bundle
41		- args = ['source-bundle', element_name, '--directory', str(tmpdir)]
42		- result = cli.run(project=project_path, args=args)
43		- result.assert_success()
44		-
45		- # Verify that the source-bundle contains our sources and a build script
46		- with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
47		- assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
48		- assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

tests/frontend/source_checkout.py

---

...	...	@@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
154	154	assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
155	155	else:
156	156	result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
	157	+
	158	+
	159	+@pytest.mark.datafiles(DATA_DIR)
	160	+def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
	161	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	162	+ element_name = 'source-bundle/source-bundle-hello.bst'
	163	+ normal_name = 'source-bundle-source-bundle-hello'
	164	+ checkout = os.path.join(str(tmpdir), 'source-checkout')
	165	+
	166	+ args = ['source-checkout', '--include-build-scripts', element_name, checkout]
	167	+ result = cli.run(project=project_path, args=args)
	168	+ result.assert_success()
	169	+
	170	+ # There sould be a script for each element (just one in this case) and a top level build script
	171	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	172	+ for script in expected_scripts:
	173	+ assert script in os.listdir(checkout)
	174	+
	175	+
	176	+@pytest.mark.datafiles(DATA_DIR)
	177	+def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
	178	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	179	+ element_name = 'source-bundle/source-bundle-hello.bst'
	180	+ normal_name = 'source-bundle-source-bundle-hello'
	181	+ tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
	182	+
	183	+ args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
	184	+ result = cli.run(project=project_path, args=args)
	185	+ result.assert_success()
	186	+
	187	+ expected_scripts = ['build.sh', 'build-' + normal_name]
	188	+
	189	+ with tarfile.open(tar_file, 'r') as tf:
	190	+ for script in expected_scripts:
	191	+ assert script in tf.getnames()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.