[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 4 commits: Add option to source-checkout to generate build scripts



Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

8 changed files:

Changes:

  • NEWS
    ... ... @@ -2,6 +2,17 @@
    2 2
     buildstream 1.3.1
    
    3 3
     =================
    
    4 4
     
    
    5
    +  o BREAKING CHANGE: The bst source-bundle command has been removed. The
    
    6
    +    functionality it provided has been replaced by the `--include-build-scripts`
    
    7
    +    option of the `bst source-checkout` command. To produce a tarball containing
    
    8
    +    an element's sources and generated build scripts you can do the command
    
    9
    +    `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
    
    10
    +
    
    11
    +  o A `bst source-checkout` command has been added. This command allows an
    
    12
    +    element's sources to be checkout out into a user specified location. For
    
    13
    +    example, `bst source-checkout foo.bst some/path/` will collect the sources
    
    14
    +    of `foo.bst` and place them in the directory `some/path`.
    
    15
    +
    
    5 16
       o All elements must now be suffixed with `.bst`
    
    6 17
         Attempting to use an element that does not have the `.bst` extension,
    
    7 18
         will result in a warning.
    

  • buildstream/_frontend/cli.py
    ... ... @@ -697,11 +697,12 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    697 697
     @click.option('--tar', 'tar', default=False, is_flag=True,
    
    698 698
                   help='Create a tarball from the element\'s sources instead of a '
    
    699 699
                        'file tree.')
    
    700
    -@click.argument('element',
    
    701
    -                type=click.Path(readable=False))
    
    700
    +@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
    
    701
    +@click.argument('element', type=click.Path(readable=False))
    
    702 702
     @click.argument('location', type=click.Path())
    
    703 703
     @click.pass_obj
    
    704
    -def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
    
    704
    +def source_checkout(app, element, location, force, deps, fetch_, except_,
    
    705
    +                    tar, build_scripts):
    
    705 706
         """Checkout sources of an element to the specified location
    
    706 707
         """
    
    707 708
         with app.initialized():
    
    ... ... @@ -711,7 +712,8 @@ def source_checkout(app, element, location, force, deps, fetch_, except_, tar):
    711 712
                                        deps=deps,
    
    712 713
                                        fetch=fetch_,
    
    713 714
                                        except_targets=except_,
    
    714
    -                                   tar=tar)
    
    715
    +                                   tar=tar,
    
    716
    +                                   include_build_scripts=build_scripts)
    
    715 717
     
    
    716 718
     
    
    717 719
     ##################################################################
    
    ... ... @@ -842,34 +844,3 @@ def workspace_list(app):
    842 844
     
    
    843 845
         with app.initialized():
    
    844 846
             app.stream.workspace_list()
    845
    -
    
    846
    -
    
    847
    -##################################################################
    
    848
    -#                     Source Bundle Command                      #
    
    849
    -##################################################################
    
    850
    -@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
    
    851
    -@click.option('--except', 'except_', multiple=True,
    
    852
    -              type=click.Path(readable=False),
    
    853
    -              help="Elements to except from the tarball")
    
    854
    -@click.option('--compression', default='gz',
    
    855
    -              type=click.Choice(['none', 'gz', 'bz2', 'xz']),
    
    856
    -              help="Compress the tar file using the given algorithm.")
    
    857
    -@click.option('--track', 'track_', default=False, is_flag=True,
    
    858
    -              help="Track new source references before bundling")
    
    859
    -@click.option('--force', '-f', default=False, is_flag=True,
    
    860
    -              help="Overwrite an existing tarball")
    
    861
    -@click.option('--directory', default=os.getcwd(),
    
    862
    -              help="The directory to write the tarball to")
    
    863
    -@click.argument('element',
    
    864
    -                type=click.Path(readable=False))
    
    865
    -@click.pass_obj
    
    866
    -def source_bundle(app, element, force, directory,
    
    867
    -                  track_, compression, except_):
    
    868
    -    """Produce a source bundle to be manually executed
    
    869
    -    """
    
    870
    -    with app.initialized():
    
    871
    -        app.stream.source_bundle(element, directory,
    
    872
    -                                 track_first=track_,
    
    873
    -                                 force=force,
    
    874
    -                                 compression=compression,
    
    875
    -                                 except_targets=except_)

  • buildstream/_stream.py
    ... ... @@ -25,7 +25,7 @@ import stat
    25 25
     import shlex
    
    26 26
     import shutil
    
    27 27
     import tarfile
    
    28
    -from contextlib import contextmanager
    
    28
    +from contextlib import contextmanager, suppress
    
    29 29
     from tempfile import TemporaryDirectory
    
    30 30
     
    
    31 31
     from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
    
    ... ... @@ -453,7 +453,8 @@ class Stream():
    453 453
                             deps='none',
    
    454 454
                             fetch=False,
    
    455 455
                             except_targets=(),
    
    456
    -                        tar=False):
    
    456
    +                        tar=False,
    
    457
    +                        include_build_scripts=False):
    
    457 458
     
    
    458 459
             self._check_location_writable(location, force=force, tar=tar)
    
    459 460
     
    
    ... ... @@ -469,10 +470,8 @@ class Stream():
    469 470
     
    
    470 471
             # Stage all sources determined by scope
    
    471 472
             try:
    
    472
    -            if tar:
    
    473
    -                self._create_source_tarball(location, elements)
    
    474
    -            else:
    
    475
    -                self._write_element_sources(location, elements)
    
    473
    +            self._source_checkout(elements, location, force, deps, fetch,
    
    474
    +                                  except_targets, tar, include_build_scripts)
    
    476 475
             except BstError as e:
    
    477 476
                 raise StreamError("Error while writing sources"
    
    478 477
                                   ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    ... ... @@ -729,87 +728,6 @@ class Stream():
    729 728
                 'workspaces': workspaces
    
    730 729
             })
    
    731 730
     
    
    732
    -    # source_bundle()
    
    733
    -    #
    
    734
    -    # Create a host buildable tarball bundle for the given target.
    
    735
    -    #
    
    736
    -    # Args:
    
    737
    -    #    target (str): The target element to bundle
    
    738
    -    #    directory (str): The directory to output the tarball
    
    739
    -    #    track_first (bool): Track new source references before bundling
    
    740
    -    #    compression (str): The compression type to use
    
    741
    -    #    force (bool): Overwrite an existing tarball
    
    742
    -    #
    
    743
    -    def source_bundle(self, target, directory, *,
    
    744
    -                      track_first=False,
    
    745
    -                      force=False,
    
    746
    -                      compression="gz",
    
    747
    -                      except_targets=()):
    
    748
    -
    
    749
    -        if track_first:
    
    750
    -            track_targets = (target,)
    
    751
    -        else:
    
    752
    -            track_targets = ()
    
    753
    -
    
    754
    -        elements, track_elements = self._load((target,), track_targets,
    
    755
    -                                              selection=PipelineSelection.ALL,
    
    756
    -                                              except_targets=except_targets,
    
    757
    -                                              track_selection=PipelineSelection.ALL,
    
    758
    -                                              fetch_subprojects=True)
    
    759
    -
    
    760
    -        # source-bundle only supports one target
    
    761
    -        target = self.targets[0]
    
    762
    -
    
    763
    -        self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
    
    764
    -
    
    765
    -        # Find the correct filename for the compression algorithm
    
    766
    -        tar_location = os.path.join(directory, target.normal_name + ".tar")
    
    767
    -        if compression != "none":
    
    768
    -            tar_location += "." + compression
    
    769
    -
    
    770
    -        # Attempt writing a file to generate a good error message
    
    771
    -        # early
    
    772
    -        #
    
    773
    -        # FIXME: A bit hackish
    
    774
    -        try:
    
    775
    -            open(tar_location, mode="x")
    
    776
    -            os.remove(tar_location)
    
    777
    -        except IOError as e:
    
    778
    -            raise StreamError("Cannot write to {0}: {1}"
    
    779
    -                              .format(tar_location, e)) from e
    
    780
    -
    
    781
    -        # Fetch and possibly track first
    
    782
    -        #
    
    783
    -        self._fetch(elements, track_elements=track_elements)
    
    784
    -
    
    785
    -        # We don't use the scheduler for this as it is almost entirely IO
    
    786
    -        # bound.
    
    787
    -
    
    788
    -        # Create a temporary directory to build the source tree in
    
    789
    -        builddir = self._context.builddir
    
    790
    -        os.makedirs(builddir, exist_ok=True)
    
    791
    -        prefix = "{}-".format(target.normal_name)
    
    792
    -
    
    793
    -        with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
    
    794
    -            source_directory = os.path.join(tempdir, 'source')
    
    795
    -            try:
    
    796
    -                os.makedirs(source_directory)
    
    797
    -            except OSError as e:
    
    798
    -                raise StreamError("Failed to create directory: {}"
    
    799
    -                                  .format(e)) from e
    
    800
    -
    
    801
    -            # Any elements that don't implement _write_script
    
    802
    -            # should not be included in the later stages.
    
    803
    -            elements = [
    
    804
    -                element for element in elements
    
    805
    -                if self._write_element_script(source_directory, element)
    
    806
    -            ]
    
    807
    -
    
    808
    -            self._write_element_sources(os.path.join(tempdir, "source"), elements)
    
    809
    -            self._write_build_script(tempdir, elements)
    
    810
    -            self._collect_sources(tempdir, tar_location,
    
    811
    -                                  target.normal_name, compression)
    
    812
    -
    
    813 731
         # redirect_element_names()
    
    814 732
         #
    
    815 733
         # Takes a list of element names and returns a list where elements have been
    
    ... ... @@ -1193,6 +1111,7 @@ class Stream():
    1193 1111
         # Helper function for source_checkout()
    
    1194 1112
         def _source_checkout(self, elements,
    
    1195 1113
                              location=None,
    
    1114
    +                         force=False,
    
    1196 1115
                              deps='none',
    
    1197 1116
                              fetch=False,
    
    1198 1117
                              except_targets=(),
    
    ... ... @@ -1203,35 +1122,20 @@ class Stream():
    1203 1122
             # Stage all our sources in a temporary directory. The this
    
    1204 1123
             # directory can be used to either construct a tarball or moved
    
    1205 1124
             # to the final desired location.
    
    1206
    -        temp_source_dir = self._create_temp_source_dir(elements, include_build_scripts)
    
    1207
    -        if tar:
    
    1208
    -            self._create_tarball(temp_source_dir.name, location)
    
    1209
    -        else:
    
    1210
    -            try:
    
    1211
    -                self._move_directory(temp_source_dir.name, location, force)
    
    1212
    -            except OSError as e:
    
    1125
    +        with TemporaryDirectory() as temp_source_dir:
    
    1126
    +            self._write_element_sources(temp_source_dir, elements)
    
    1127
    +            if include_build_scripts:
    
    1128
    +                self._write_build_scripts(temp_source_dir, elements)
    
    1129
    +
    
    1130
    +            if tar:
    
    1131
    +                self._create_tarball(temp_source_dir, location)
    
    1132
    +            else:
    
    1213 1133
                     try:
    
    1214
    -                    shutil.rmtree(location)
    
    1215
    -                except FileNotFoundError:
    
    1216
    -                    pass
    
    1217
    -                temp_source_dir.cleanup()
    
    1218
    -                raise StreamError("Failed to checkout sources to {}: {}".format(location, e))
    
    1219
    -
    
    1220
    -        # Ensure the temporary directory is cleaned up. If it has been
    
    1221
    -        # moved temp_source_dir will no longer exist with it's
    
    1222
    -        # original name. This is expected.
    
    1223
    -        try:
    
    1224
    -            temp_source_dir.cleanup()
    
    1225
    -        except FileNotFoundError:
    
    1226
    -            pass
    
    1227
    -
    
    1228
    -    # Construct a TemporaryDirectory containing the sources of elements.
    
    1229
    -    def _create_temp_source_dir(self, elements, include_build_scripts):
    
    1230
    -        tempdir = TemporaryDirectory()
    
    1231
    -        self._write_element_sources(tempdir.name, elements)
    
    1232
    -        if include_build_scripts:
    
    1233
    -            self._write_build_scripts(tempdir.name, elements)
    
    1234
    -        return tempdir
    
    1134
    +                    self._move_directory(temp_source_dir, location, force)
    
    1135
    +                except OSError as e:
    
    1136
    +                    with suppress(FileNotFoundError):
    
    1137
    +                        shutil.rmtree(location)
    
    1138
    +                    raise StreamError("Failed to checkout sources to {}: {}".format(location, e))
    
    1235 1139
     
    
    1236 1140
         # Utility to move the contents of one directory into the specified destination.
    
    1237 1141
         # This is used because os.rename does not work when the target and destination
    
    ... ... @@ -1263,17 +1167,33 @@ class Stream():
    1263 1167
                     os.makedirs(element_source_dir)
    
    1264 1168
                     element._stage_sources_at(element_source_dir)
    
    1265 1169
     
    
    1266
    -    # Create a tarball containing the sources of each element in elements
    
    1267
    -    def _create_source_tarball(self, directory, elements):
    
    1268
    -        with tarfile.open(name=directory, mode='w') as tf:
    
    1269
    -            with TemporaryDirectory() as tmpdir:
    
    1270
    -                self._write_element_sources(tmpdir, elements)
    
    1271
    -                for item in os.listdir(tmpdir):
    
    1272
    -                    file_to_add = os.path.join(tmpdir, item)
    
    1273
    -                    tf.add(file_to_add, arcname=item)
    
    1170
    +    # Create a tarball from the content of directory
    
    1171
    +    def _create_tarball(self, directory, tar_name):
    
    1172
    +        try:
    
    1173
    +            with utils.save_file_atomic(tar_name, mode='wb') as f:
    
    1174
    +                # This TarFile does not need to be explicitly closed
    
    1175
    +                # as the underlying file object will be closed be the
    
    1176
    +                # save_file_atomic contect manager
    
    1177
    +                tarball = tarfile.open(fileobj=f, mode='w')
    
    1178
    +                for item in os.listdir(str(directory)):
    
    1179
    +                    file_to_add = os.path.join(directory, item)
    
    1180
    +                    tarball.add(file_to_add, arcname=item)
    
    1181
    +        except OSError as e:
    
    1182
    +            # If we have a partially constructed tar file, clean up after ourselves
    
    1183
    +            try:
    
    1184
    +                os.remove(tar_name)
    
    1185
    +            except OSError:
    
    1186
    +                pass
    
    1187
    +            raise StreamError("Failed to create tar archive: {}".format(e)) from e
    
    1188
    +
    
    1189
    +    # Write all the build_scripts for elements in the directory location
    
    1190
    +    def _write_build_scripts(self, location, elements):
    
    1191
    +        for element in elements:
    
    1192
    +            self._write_element_script(location, element)
    
    1193
    +        self._write_master_build_script(location, elements)
    
    1274 1194
     
    
    1275 1195
         # Write a master build script to the sandbox
    
    1276
    -    def _write_build_script(self, directory, elements):
    
    1196
    +    def _write_master_build_script(self, directory, elements):
    
    1277 1197
     
    
    1278 1198
             module_string = ""
    
    1279 1199
             for element in elements:
    

  • doc/source/using_commands.rst
    ... ... @@ -86,13 +86,6 @@ project's main directory.
    86 86
     
    
    87 87
     ----
    
    88 88
     
    
    89
    -.. _invoking_source_bundle:
    
    90
    -
    
    91
    -.. click:: buildstream._frontend.cli:source_bundle
    
    92
    -   :prog: bst source bundle
    
    93
    -
    
    94
    -----
    
    95
    -
    
    96 89
     .. _invoking_workspace:
    
    97 90
     
    
    98 91
     .. click:: buildstream._frontend.cli:workspace
    

  • tests/completions/completions.py
    ... ... @@ -16,7 +16,6 @@ MAIN_COMMANDS = [
    16 16
         'shell ',
    
    17 17
         'show ',
    
    18 18
         'source-checkout ',
    
    19
    -    'source-bundle ',
    
    20 19
         'track ',
    
    21 20
         'workspace '
    
    22 21
     ]
    

  • tests/frontend/help.py
    ... ... @@ -25,7 +25,6 @@ def test_help_main(cli):
    25 25
         ('push'),
    
    26 26
         ('shell'),
    
    27 27
         ('show'),
    
    28
    -    ('source-bundle'),
    
    29 28
         ('track'),
    
    30 29
         ('workspace')
    
    31 30
     ])
    

  • tests/frontend/source_bundle.py deleted
    1
    -#
    
    2
    -#  Copyright (C) 2018 Bloomberg Finance LP
    
    3
    -#
    
    4
    -#  This program is free software; you can redistribute it and/or
    
    5
    -#  modify it under the terms of the GNU Lesser General Public
    
    6
    -#  License as published by the Free Software Foundation; either
    
    7
    -#  version 2 of the License, or (at your option) any later version.
    
    8
    -#
    
    9
    -#  This library is distributed in the hope that it will be useful,
    
    10
    -#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    -#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    
    12
    -#  Lesser General Public License for more details.
    
    13
    -#
    
    14
    -#  You should have received a copy of the GNU Lesser General Public
    
    15
    -#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    -#
    
    17
    -#  Authors: Chandan Singh <csingh43 bloomberg net>
    
    18
    -#
    
    19
    -
    
    20
    -import os
    
    21
    -import tarfile
    
    22
    -
    
    23
    -import pytest
    
    24
    -
    
    25
    -from tests.testutils import cli
    
    26
    -
    
    27
    -# Project directory
    
    28
    -DATA_DIR = os.path.join(
    
    29
    -    os.path.dirname(os.path.realpath(__file__)),
    
    30
    -    "project",
    
    31
    -)
    
    32
    -
    
    33
    -
    
    34
    -@pytest.mark.datafiles(DATA_DIR)
    
    35
    -def test_source_bundle(cli, tmpdir, datafiles):
    
    36
    -    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    37
    -    element_name = 'source-bundle/source-bundle-hello.bst'
    
    38
    -    normal_name = 'source-bundle-source-bundle-hello'
    
    39
    -
    
    40
    -    # Verify that we can correctly produce a source-bundle
    
    41
    -    args = ['source-bundle', element_name, '--directory', str(tmpdir)]
    
    42
    -    result = cli.run(project=project_path, args=args)
    
    43
    -    result.assert_success()
    
    44
    -
    
    45
    -    # Verify that the source-bundle contains our sources and a build script
    
    46
    -    with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
    
    47
    -        assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
    
    48
    -        assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

  • tests/frontend/source_checkout.py
    ... ... @@ -154,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
    154 154
             assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
    
    155 155
         else:
    
    156 156
             result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
    
    157
    +
    
    158
    +
    
    159
    +@pytest.mark.datafiles(DATA_DIR)
    
    160
    +def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
    
    161
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    162
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    163
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    164
    +    checkout = os.path.join(str(tmpdir), 'source-checkout')
    
    165
    +
    
    166
    +    args = ['source-checkout', '--include-build-scripts', element_name, checkout]
    
    167
    +    result = cli.run(project=project_path, args=args)
    
    168
    +    result.assert_success()
    
    169
    +
    
    170
    +    # There sould be a script for each element (just one in this case) and a top level build script
    
    171
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    172
    +    for script in expected_scripts:
    
    173
    +        assert script in os.listdir(checkout)
    
    174
    +
    
    175
    +
    
    176
    +@pytest.mark.datafiles(DATA_DIR)
    
    177
    +def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
    
    178
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    179
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    180
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    181
    +    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
    
    182
    +
    
    183
    +    args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
    
    184
    +    result = cli.run(project=project_path, args=args)
    
    185
    +    result.assert_success()
    
    186
    +
    
    187
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    188
    +
    
    189
    +    with tarfile.open(tar_file, 'r') as tf:
    
    190
    +        for script in expected_scripts:
    
    191
    +            assert script in tf.getnames()



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]