Chandan Singh pushed to branch chandan/source-checkout at BuildStream / buildstream
Commits:
-
b1ad58d0
by Chandan Singh at 2018-09-22T19:11:20Z
6 changed files:
- buildstream/_frontend/cli.py
- buildstream/_stream.py
- tests/completions/completions.py
- + tests/frontend/project/elements/checkout-deps.bst
- + tests/frontend/project/files/etc-files/etc/buildstream/config
- + tests/frontend/source_checkout.py
Changes:
... | ... | @@ -662,6 +662,30 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar): |
662 | 662 |
tar=tar)
|
663 | 663 |
|
664 | 664 |
|
665 |
+##################################################################
|
|
666 |
+# Source Checkout Command #
|
|
667 |
+##################################################################
|
|
668 |
+@cli.command(name="source-checkout", short_help="Checkout sources for an element")
|
|
669 |
+@click.option('--except', 'except_', multiple=True,
|
|
670 |
+ type=click.Path(readable=False),
|
|
671 |
+ help="Except certain dependencies")
|
|
672 |
+@click.option('--deps', '-d', default='none',
|
|
673 |
+ type=click.Choice(['build', 'none', 'run', 'all']),
|
|
674 |
+ help='The dependencies whose sources to checkout (default: none)')
|
|
675 |
+@click.argument('element',
|
|
676 |
+ type=click.Path(readable=False))
|
|
677 |
+@click.argument('location', type=click.Path())
|
|
678 |
+@click.pass_obj
|
|
679 |
+def source_checkout(app, element, location, deps, except_):
|
|
680 |
+ """Checkout sources of an element to the specified location
|
|
681 |
+ """
|
|
682 |
+ with app.initialized():
|
|
683 |
+ app.stream.source_checkout(element,
|
|
684 |
+ location=location,
|
|
685 |
+ deps=deps,
|
|
686 |
+ except_targets=except_)
|
|
687 |
+ |
|
688 |
+ |
|
665 | 689 |
##################################################################
|
666 | 690 |
# Workspace Command #
|
667 | 691 |
##################################################################
|
... | ... | @@ -381,27 +381,7 @@ class Stream(): |
381 | 381 |
elements, _ = self._load((target,), (), fetch_subprojects=True)
|
382 | 382 |
target = elements[0]
|
383 | 383 |
|
384 |
- if not tar:
|
|
385 |
- try:
|
|
386 |
- os.makedirs(location, exist_ok=True)
|
|
387 |
- except OSError as e:
|
|
388 |
- raise StreamError("Failed to create checkout directory: '{}'"
|
|
389 |
- .format(e)) from e
|
|
390 |
- |
|
391 |
- if not tar:
|
|
392 |
- if not os.access(location, os.W_OK):
|
|
393 |
- raise StreamError("Checkout directory '{}' not writable"
|
|
394 |
- .format(location))
|
|
395 |
- if not force and os.listdir(location):
|
|
396 |
- raise StreamError("Checkout directory '{}' not empty"
|
|
397 |
- .format(location))
|
|
398 |
- elif os.path.exists(location) and location != '-':
|
|
399 |
- if not os.access(location, os.W_OK):
|
|
400 |
- raise StreamError("Output file '{}' not writable"
|
|
401 |
- .format(location))
|
|
402 |
- if not force and os.path.exists(location):
|
|
403 |
- raise StreamError("Output file '{}' already exists"
|
|
404 |
- .format(location))
|
|
384 |
+ self.__check_location_writable(location, force=force, tar=tar)
|
|
405 | 385 |
|
406 | 386 |
# Stage deps into a temporary sandbox first
|
407 | 387 |
try:
|
... | ... | @@ -438,6 +418,35 @@ class Stream(): |
438 | 418 |
raise StreamError("Error while staging dependencies into a sandbox"
|
439 | 419 |
": '{}'".format(e), detail=e.detail, reason=e.reason) from e
|
440 | 420 |
|
421 |
+ # source_checkout()
|
|
422 |
+ #
|
|
423 |
+ # Checkout sources of the target element to the specified location
|
|
424 |
+ #
|
|
425 |
+ # Args:
|
|
426 |
+ # target (str): The target element whose sources to checkout
|
|
427 |
+ # location (str): Location to checkout the sources to
|
|
428 |
+ # deps (str): The dependencies to checkout
|
|
429 |
+ # except_targets (list): List of targets to except from staging
|
|
430 |
+ #
|
|
431 |
+ def source_checkout(self, target, *,
|
|
432 |
+ location=None,
|
|
433 |
+ deps='none',
|
|
434 |
+ except_targets=()):
|
|
435 |
+ |
|
436 |
+ self.__check_location_writable(location)
|
|
437 |
+ |
|
438 |
+ elements, _ = self._load((target,), (),
|
|
439 |
+ selection=deps,
|
|
440 |
+ except_targets=except_targets,
|
|
441 |
+ fetch_subprojects=True)
|
|
442 |
+ |
|
443 |
+ # Stage all sources determined by scope
|
|
444 |
+ try:
|
|
445 |
+ self._write_element_sources(location, elements)
|
|
446 |
+ except BstError as e:
|
|
447 |
+ raise StreamError("Error while staging sources"
|
|
448 |
+ ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
|
|
449 |
+ |
|
441 | 450 |
# workspace_open
|
442 | 451 |
#
|
443 | 452 |
# Open a project workspace
|
... | ... | @@ -721,7 +730,7 @@ class Stream(): |
721 | 730 |
if self._write_element_script(source_directory, element)
|
722 | 731 |
]
|
723 | 732 |
|
724 |
- self._write_element_sources(tempdir, elements)
|
|
733 |
+ self._write_element_sources(os.path.join(tempdir, "source"), elements)
|
|
725 | 734 |
self._write_build_script(tempdir, elements)
|
726 | 735 |
self._collect_sources(tempdir, tar_location,
|
727 | 736 |
target.normal_name, compression)
|
... | ... | @@ -1084,11 +1093,10 @@ class Stream(): |
1084 | 1093 |
# Write all source elements to the given directory
|
1085 | 1094 |
def _write_element_sources(self, directory, elements):
|
1086 | 1095 |
for element in elements:
|
1087 |
- source_dir = os.path.join(directory, "source")
|
|
1088 |
- element_source_dir = os.path.join(source_dir, element.normal_name)
|
|
1089 |
- os.makedirs(element_source_dir)
|
|
1090 |
- |
|
1091 |
- element._stage_sources_at(element_source_dir)
|
|
1096 |
+ element_source_dir = os.path.join(directory, element.normal_name)
|
|
1097 |
+ if list(element.sources()):
|
|
1098 |
+ os.makedirs(element_source_dir)
|
|
1099 |
+ element._stage_sources_at(element_source_dir)
|
|
1092 | 1100 |
|
1093 | 1101 |
# Write a master build script to the sandbox
|
1094 | 1102 |
def _write_build_script(self, directory, elements):
|
... | ... | @@ -1117,3 +1125,29 @@ class Stream(): |
1117 | 1125 |
|
1118 | 1126 |
with tarfile.open(tar_name, permissions) as tar:
|
1119 | 1127 |
tar.add(directory, arcname=element_name)
|
1128 |
+ |
|
1129 |
+ #############################################################
|
|
1130 |
+ # Private Methods #
|
|
1131 |
+ #############################################################
|
|
1132 |
+ |
|
1133 |
+ # Check if given location is writable
|
|
1134 |
+ def __check_location_writable(self, location, force=False, tar=False):
|
|
1135 |
+ if not tar:
|
|
1136 |
+ try:
|
|
1137 |
+ os.makedirs(location, exist_ok=True)
|
|
1138 |
+ except OSError as e:
|
|
1139 |
+ raise StreamError("Failed to create checkout directory: '{}'"
|
|
1140 |
+ .format(e)) from e
|
|
1141 |
+ if not os.access(location, os.W_OK):
|
|
1142 |
+ raise StreamError("Checkout directory '{}' not writable"
|
|
1143 |
+ .format(location))
|
|
1144 |
+ if not force and os.listdir(location):
|
|
1145 |
+ raise StreamError("Checkout directory '{}' not empty"
|
|
1146 |
+ .format(location))
|
|
1147 |
+ elif os.path.exists(location) and location != '-':
|
|
1148 |
+ if not os.access(location, os.W_OK):
|
|
1149 |
+ raise StreamError("Output file '{}' not writable"
|
|
1150 |
+ .format(location))
|
|
1151 |
+ if not force and os.path.exists(location):
|
|
1152 |
+ raise StreamError("Output file '{}' already exists"
|
|
1153 |
+ .format(location))
|
... | ... | @@ -15,6 +15,7 @@ MAIN_COMMANDS = [ |
15 | 15 |
'push ',
|
16 | 16 |
'shell ',
|
17 | 17 |
'show ',
|
18 |
+ 'source-checkout',
|
|
18 | 19 |
'source-bundle ',
|
19 | 20 |
'track ',
|
20 | 21 |
'workspace '
|
1 |
+kind: import
|
|
2 |
+description: It is important for this element to have both build and runtime dependencies
|
|
3 |
+sources:
|
|
4 |
+- kind: local
|
|
5 |
+ path: files/etc-files
|
|
6 |
+depends:
|
|
7 |
+- filename: import-dev.bst
|
|
8 |
+ type: build
|
|
9 |
+- filename: import-bin.bst
|
|
10 |
+ type: runtime
|
1 |
+config
|
1 |
+import os
|
|
2 |
+import pytest
|
|
3 |
+ |
|
4 |
+from tests.testutils import cli
|
|
5 |
+ |
|
6 |
+# Project directory
|
|
7 |
+DATA_DIR = os.path.join(
|
|
8 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
9 |
+ "project",
|
|
10 |
+)
|
|
11 |
+ |
|
12 |
+ |
|
13 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
14 |
+def test_source_checkout(datafiles, cli):
|
|
15 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
16 |
+ checkout = os.path.join(cli.directory, 'source-checkout')
|
|
17 |
+ target = 'checkout-deps.bst'
|
|
18 |
+ |
|
19 |
+ result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
|
|
20 |
+ result.assert_success()
|
|
21 |
+ |
|
22 |
+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
|
|
23 |
+ |
|
24 |
+ |
|
25 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
26 |
+@pytest.mark.parametrize('deps', [("build"), ("none"), ("run"), ("all")])
|
|
27 |
+def test_source_checkout_deps(datafiles, cli, deps):
|
|
28 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
29 |
+ checkout = os.path.join(cli.directory, 'source-checkout')
|
|
30 |
+ target = 'checkout-deps.bst'
|
|
31 |
+ |
|
32 |
+ result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
|
|
33 |
+ result.assert_success()
|
|
34 |
+ |
|
35 |
+ # Sources of the target
|
|
36 |
+ if deps == 'build':
|
|
37 |
+ assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
|
|
38 |
+ else:
|
|
39 |
+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
|
|
40 |
+ |
|
41 |
+ # Sources of the target's build dependencies
|
|
42 |
+ if deps in ('build', 'all'):
|
|
43 |
+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
|
|
44 |
+ else:
|
|
45 |
+ assert not os.path.exists(os.path.join(checkout, 'import-dev'))
|
|
46 |
+ |
|
47 |
+ # Sources of the target's runtime dependencies
|
|
48 |
+ if deps in ('run', 'all'):
|
|
49 |
+ assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
|
|
50 |
+ else:
|
|
51 |
+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
|
|
52 |
+ |
|
53 |
+ |
|
54 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
55 |
+def test_source_checkout_except(datafiles, cli):
|
|
56 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
57 |
+ checkout = os.path.join(cli.directory, 'source-checkout')
|
|
58 |
+ target = 'checkout-deps.bst'
|
|
59 |
+ |
|
60 |
+ result = cli.run(project=project, args=['source-checkout', target,
|
|
61 |
+ '--deps', 'all',
|
|
62 |
+ '--except', 'import-bin.bst',
|
|
63 |
+ checkout])
|
|
64 |
+ result.assert_success()
|
|
65 |
+ |
|
66 |
+ # Sources for the target should be present
|
|
67 |
+ assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
|
|
68 |
+ |
|
69 |
+ # Sources for import-bin.bst should not be present
|
|
70 |
+ assert not os.path.exists(os.path.join(checkout, 'import-bin'))
|
|
71 |
+ |
|
72 |
+ # Sources for other dependencies should be present
|
|
73 |
+ assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
|