[Notes] [Git][BuildStream/buildstream][phil/source-checkout-options] 5 commits: Add --force / -f option to bst source-checkout command



Title: GitLab

Phil Dawson pushed to branch phil/source-checkout-options at BuildStream / buildstream

Commits:

8 changed files:

Changes:

  • NEWS
    ... ... @@ -2,6 +2,17 @@
    2 2
     buildstream 1.3.1
    
    3 3
     =================
    
    4 4
     
    
    5
    +  o BREAKING CHANGE: The bst source-bundle command has been removed. The
    
    6
    +    functionality it provided has been replaced by the `--include-build-scripts`
    
    7
    +    option of the `bst source-checkout` command. To produce a tarball containing
    
    8
    +    an element's sources and generated build scripts you can do the command
    
    9
    +    `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
    
    10
    +
    
    11
    +  o A `bst source-checkout` command has been added. This command allows an
    
    12
    +    element's sources to be checkout out into a user specified location. For
    
    13
    +    example, `bst source-checkout foo.bst some/path/` will collect the sources
    
    14
    +    of `foo.bst` and place them in the directory `some/path`.
    
    15
    +
    
    5 16
       o BREAKING CHANGE: The 'manual' element lost its default 'MAKEFLAGS' and 'V'
    
    6 17
         environment variables. There is already a 'make' element with the same
    
    7 18
         variables. Note that this is a breaking change, it will require users to
    

  • buildstream/_frontend/cli.py
    ... ... @@ -668,6 +668,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    668 668
     #                  Source Checkout Command                      #
    
    669 669
     ##################################################################
    
    670 670
     @cli.command(name='source-checkout', short_help='Checkout sources for an element')
    
    671
    +@click.option('--force', '-f', default=False, is_flag=True,
    
    672
    +              help="Allow files to be overwritten")
    
    671 673
     @click.option('--except', 'except_', multiple=True,
    
    672 674
                   type=click.Path(readable=False),
    
    673 675
                   help="Except certain dependencies")
    
    ... ... @@ -679,20 +681,23 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    679 681
     @click.option('--tar', 'tar', default=False, is_flag=True,
    
    680 682
                   help='Create a tarball from the element\'s sources instead of a '
    
    681 683
                        'file tree.')
    
    682
    -@click.argument('element',
    
    683
    -                type=click.Path(readable=False))
    
    684
    +@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
    
    685
    +@click.argument('element', type=click.Path(readable=False))
    
    684 686
     @click.argument('location', type=click.Path())
    
    685 687
     @click.pass_obj
    
    686
    -def source_checkout(app, element, location, deps, fetch_, except_, tar):
    
    688
    +def source_checkout(app, element, location, force, deps, fetch_, except_,
    
    689
    +                    tar, build_scripts):
    
    687 690
         """Checkout sources of an element to the specified location
    
    688 691
         """
    
    689 692
         with app.initialized():
    
    690 693
             app.stream.source_checkout(element,
    
    691 694
                                        location=location,
    
    695
    +                                   force=force,
    
    692 696
                                        deps=deps,
    
    693 697
                                        fetch=fetch_,
    
    694 698
                                        except_targets=except_,
    
    695
    -                                   tar=tar)
    
    699
    +                                   tar=tar,
    
    700
    +                                   include_build_scripts=build_scripts)
    
    696 701
     
    
    697 702
     
    
    698 703
     ##################################################################
    
    ... ... @@ -831,34 +836,3 @@ def workspace_list(app):
    831 836
     
    
    832 837
         with app.initialized():
    
    833 838
             app.stream.workspace_list()
    834
    -
    
    835
    -
    
    836
    -##################################################################
    
    837
    -#                     Source Bundle Command                      #
    
    838
    -##################################################################
    
    839
    -@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
    
    840
    -@click.option('--except', 'except_', multiple=True,
    
    841
    -              type=click.Path(readable=False),
    
    842
    -              help="Elements to except from the tarball")
    
    843
    -@click.option('--compression', default='gz',
    
    844
    -              type=click.Choice(['none', 'gz', 'bz2', 'xz']),
    
    845
    -              help="Compress the tar file using the given algorithm.")
    
    846
    -@click.option('--track', 'track_', default=False, is_flag=True,
    
    847
    -              help="Track new source references before bundling")
    
    848
    -@click.option('--force', '-f', default=False, is_flag=True,
    
    849
    -              help="Overwrite an existing tarball")
    
    850
    -@click.option('--directory', default=os.getcwd(),
    
    851
    -              help="The directory to write the tarball to")
    
    852
    -@click.argument('element',
    
    853
    -                type=click.Path(readable=False))
    
    854
    -@click.pass_obj
    
    855
    -def source_bundle(app, element, force, directory,
    
    856
    -                  track_, compression, except_):
    
    857
    -    """Produce a source bundle to be manually executed
    
    858
    -    """
    
    859
    -    with app.initialized():
    
    860
    -        app.stream.source_bundle(element, directory,
    
    861
    -                                 track_first=track_,
    
    862
    -                                 force=force,
    
    863
    -                                 compression=compression,
    
    864
    -                                 except_targets=except_)

  • buildstream/_stream.py
    ... ... @@ -436,12 +436,14 @@ class Stream():
    436 436
         #
    
    437 437
         def source_checkout(self, target, *,
    
    438 438
                             location=None,
    
    439
    +                        force=False,
    
    439 440
                             deps='none',
    
    440 441
                             fetch=False,
    
    441 442
                             except_targets=(),
    
    442
    -                        tar=False):
    
    443
    +                        tar=False,
    
    444
    +                        include_build_scripts=False):
    
    443 445
     
    
    444
    -        self._check_location_writable(location, tar=tar)
    
    446
    +        self._check_location_writable(location, force=force, tar=tar)
    
    445 447
     
    
    446 448
             elements, _ = self._load((target,), (),
    
    447 449
                                      selection=deps,
    
    ... ... @@ -455,10 +457,8 @@ class Stream():
    455 457
     
    
    456 458
             # Stage all sources determined by scope
    
    457 459
             try:
    
    458
    -            if tar:
    
    459
    -                self._create_source_tarball(location, elements)
    
    460
    -            else:
    
    461
    -                self._write_element_sources(location, elements)
    
    460
    +            self._source_checkout(elements, location, force, deps, fetch,
    
    461
    +                                  except_targets, tar, include_build_scripts)
    
    462 462
             except BstError as e:
    
    463 463
                 raise StreamError("Error while writing sources"
    
    464 464
                                   ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    ... ... @@ -670,87 +670,6 @@ class Stream():
    670 670
                 'workspaces': workspaces
    
    671 671
             })
    
    672 672
     
    
    673
    -    # source_bundle()
    
    674
    -    #
    
    675
    -    # Create a host buildable tarball bundle for the given target.
    
    676
    -    #
    
    677
    -    # Args:
    
    678
    -    #    target (str): The target element to bundle
    
    679
    -    #    directory (str): The directory to output the tarball
    
    680
    -    #    track_first (bool): Track new source references before bundling
    
    681
    -    #    compression (str): The compression type to use
    
    682
    -    #    force (bool): Overwrite an existing tarball
    
    683
    -    #
    
    684
    -    def source_bundle(self, target, directory, *,
    
    685
    -                      track_first=False,
    
    686
    -                      force=False,
    
    687
    -                      compression="gz",
    
    688
    -                      except_targets=()):
    
    689
    -
    
    690
    -        if track_first:
    
    691
    -            track_targets = (target,)
    
    692
    -        else:
    
    693
    -            track_targets = ()
    
    694
    -
    
    695
    -        elements, track_elements = self._load((target,), track_targets,
    
    696
    -                                              selection=PipelineSelection.ALL,
    
    697
    -                                              except_targets=except_targets,
    
    698
    -                                              track_selection=PipelineSelection.ALL,
    
    699
    -                                              fetch_subprojects=True)
    
    700
    -
    
    701
    -        # source-bundle only supports one target
    
    702
    -        target = self.targets[0]
    
    703
    -
    
    704
    -        self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
    
    705
    -
    
    706
    -        # Find the correct filename for the compression algorithm
    
    707
    -        tar_location = os.path.join(directory, target.normal_name + ".tar")
    
    708
    -        if compression != "none":
    
    709
    -            tar_location += "." + compression
    
    710
    -
    
    711
    -        # Attempt writing a file to generate a good error message
    
    712
    -        # early
    
    713
    -        #
    
    714
    -        # FIXME: A bit hackish
    
    715
    -        try:
    
    716
    -            open(tar_location, mode="x")
    
    717
    -            os.remove(tar_location)
    
    718
    -        except IOError as e:
    
    719
    -            raise StreamError("Cannot write to {0}: {1}"
    
    720
    -                              .format(tar_location, e)) from e
    
    721
    -
    
    722
    -        # Fetch and possibly track first
    
    723
    -        #
    
    724
    -        self._fetch(elements, track_elements=track_elements)
    
    725
    -
    
    726
    -        # We don't use the scheduler for this as it is almost entirely IO
    
    727
    -        # bound.
    
    728
    -
    
    729
    -        # Create a temporary directory to build the source tree in
    
    730
    -        builddir = self._context.builddir
    
    731
    -        os.makedirs(builddir, exist_ok=True)
    
    732
    -        prefix = "{}-".format(target.normal_name)
    
    733
    -
    
    734
    -        with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
    
    735
    -            source_directory = os.path.join(tempdir, 'source')
    
    736
    -            try:
    
    737
    -                os.makedirs(source_directory)
    
    738
    -            except OSError as e:
    
    739
    -                raise StreamError("Failed to create directory: {}"
    
    740
    -                                  .format(e)) from e
    
    741
    -
    
    742
    -            # Any elements that don't implement _write_script
    
    743
    -            # should not be included in the later stages.
    
    744
    -            elements = [
    
    745
    -                element for element in elements
    
    746
    -                if self._write_element_script(source_directory, element)
    
    747
    -            ]
    
    748
    -
    
    749
    -            self._write_element_sources(os.path.join(tempdir, "source"), elements)
    
    750
    -            self._write_build_script(tempdir, elements)
    
    751
    -            self._collect_sources(tempdir, tar_location,
    
    752
    -                                  target.normal_name, compression)
    
    753
    -
    
    754 673
         # redirect_element_names()
    
    755 674
         #
    
    756 675
         # Takes a list of element names and returns a list where elements have been
    
    ... ... @@ -1134,6 +1053,7 @@ class Stream():
    1134 1053
         # Helper function for source_checkout()
    
    1135 1054
         def _source_checkout(self, elements,
    
    1136 1055
                              location=None,
    
    1056
    +                         force=False,
    
    1137 1057
                              deps='none',
    
    1138 1058
                              fetch=False,
    
    1139 1059
                              except_targets=(),
    
    ... ... @@ -1149,8 +1069,12 @@ class Stream():
    1149 1069
                 self._create_tarball(temp_source_dir.name, location)
    
    1150 1070
             else:
    
    1151 1071
                 try:
    
    1152
    -                shutil.move(temp_source_dir.name, location)
    
    1072
    +                self._move_directory(temp_source_dir.name, location, force)
    
    1153 1073
                 except OSError as e:
    
    1074
    +                try:
    
    1075
    +                    shutil.rmtree(location)
    
    1076
    +                except FileNotFoundError:
    
    1077
    +                    pass
    
    1154 1078
                     temp_source_dir.cleanup()
    
    1155 1079
                     raise StreamError("Failed to checkout sources to {}: {}".format(location, e))
    
    1156 1080
     
    
    ... ... @@ -1170,6 +1094,20 @@ class Stream():
    1170 1094
                 self._write_build_scripts(tempdir.name, elements)
    
    1171 1095
             return tempdir
    
    1172 1096
     
    
    1097
    +    # Utility to move the contents of one directory into the specified destination.
    
    1098
    +    # This is used because os.rename does not work when the target and destination
    
    1099
    +    # are on different devices and shutil.move does not behave in the desired way.
    
    1100
    +    def _move_directory(self, target, destination, force=False):
    
    1101
    +        if force:
    
    1102
    +            try:
    
    1103
    +                shutil.rmtree(destination)
    
    1104
    +            except FileNotFoundError:
    
    1105
    +                pass
    
    1106
    +
    
    1107
    +        os.makedirs(destination, exist_ok=True)
    
    1108
    +        for item in os.listdir(str(target)):
    
    1109
    +            shutil.move(os.path.join(target, item), destination)
    
    1110
    +
    
    1173 1111
     
    
    1174 1112
         # Write the element build script to the given directory
    
    1175 1113
         def _write_element_script(self, directory, element):
    
    ... ... @@ -1187,17 +1125,29 @@ class Stream():
    1187 1125
                     os.makedirs(element_source_dir)
    
    1188 1126
                     element._stage_sources_at(element_source_dir)
    
    1189 1127
     
    
    1190
    -    # Create a tarball containing the sources of each element in elements
    
    1191
    -    def _create_source_tarball(self, directory, elements):
    
    1192
    -        with tarfile.open(name=directory, mode='w') as tf:
    
    1193
    -            with TemporaryDirectory() as tmpdir:
    
    1194
    -                self._write_element_sources(tmpdir, elements)
    
    1195
    -                for item in os.listdir(tmpdir):
    
    1196
    -                    file_to_add = os.path.join(tmpdir, item)
    
    1128
    +    # Create a tarball from the content of directory
    
    1129
    +    def _create_tarball(self, directory, tar_name):
    
    1130
    +        try:
    
    1131
    +            with tarfile.open(name=tar_name, mode='w') as tf:
    
    1132
    +                for item in os.listdir(str(directory)):
    
    1133
    +                    file_to_add = os.path.join(directory, item)
    
    1197 1134
                         tf.add(file_to_add, arcname=item)
    
    1135
    +        except OSError as e:
    
    1136
    +            # If we have a partially constructed tar file, clean up after ourselves
    
    1137
    +            try:
    
    1138
    +                os.remove(tar_name)
    
    1139
    +            except OSError:
    
    1140
    +                pass
    
    1141
    +            raise StreamError("Failed to create tar archive: {}".format(e)) from e
    
    1142
    +
    
    1143
    +    # Write all the build_scripts for elements in the directory location
    
    1144
    +    def _write_build_scripts(self, location, elements):
    
    1145
    +        for element in elements:
    
    1146
    +            self._write_element_script(location, element)
    
    1147
    +        self._write_master_build_script(location, elements)
    
    1198 1148
     
    
    1199 1149
         # Write a master build script to the sandbox
    
    1200
    -    def _write_build_script(self, directory, elements):
    
    1150
    +    def _write_master_build_script(self, directory, elements):
    
    1201 1151
     
    
    1202 1152
             module_string = ""
    
    1203 1153
             for element in elements:
    

  • doc/source/using_commands.rst
    ... ... @@ -86,13 +86,6 @@ project's main directory.
    86 86
     
    
    87 87
     ----
    
    88 88
     
    
    89
    -.. _invoking_source_bundle:
    
    90
    -
    
    91
    -.. click:: buildstream._frontend.cli:source_bundle
    
    92
    -   :prog: bst source bundle
    
    93
    -
    
    94
    -----
    
    95
    -
    
    96 89
     .. _invoking_workspace:
    
    97 90
     
    
    98 91
     .. click:: buildstream._frontend.cli:workspace
    

  • tests/completions/completions.py
    ... ... @@ -16,7 +16,6 @@ MAIN_COMMANDS = [
    16 16
         'shell ',
    
    17 17
         'show ',
    
    18 18
         'source-checkout ',
    
    19
    -    'source-bundle ',
    
    20 19
         'track ',
    
    21 20
         'workspace '
    
    22 21
     ]
    

  • tests/frontend/help.py
    ... ... @@ -25,7 +25,6 @@ def test_help_main(cli):
    25 25
         ('push'),
    
    26 26
         ('shell'),
    
    27 27
         ('show'),
    
    28
    -    ('source-bundle'),
    
    29 28
         ('track'),
    
    30 29
         ('workspace')
    
    31 30
     ])
    

  • tests/frontend/source_bundle.py deleted
    1
    -#
    
    2
    -#  Copyright (C) 2018 Bloomberg Finance LP
    
    3
    -#
    
    4
    -#  This program is free software; you can redistribute it and/or
    
    5
    -#  modify it under the terms of the GNU Lesser General Public
    
    6
    -#  License as published by the Free Software Foundation; either
    
    7
    -#  version 2 of the License, or (at your option) any later version.
    
    8
    -#
    
    9
    -#  This library is distributed in the hope that it will be useful,
    
    10
    -#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    -#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    
    12
    -#  Lesser General Public License for more details.
    
    13
    -#
    
    14
    -#  You should have received a copy of the GNU Lesser General Public
    
    15
    -#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    -#
    
    17
    -#  Authors: Chandan Singh <csingh43 bloomberg net>
    
    18
    -#
    
    19
    -
    
    20
    -import os
    
    21
    -import tarfile
    
    22
    -
    
    23
    -import pytest
    
    24
    -
    
    25
    -from tests.testutils import cli
    
    26
    -
    
    27
    -# Project directory
    
    28
    -DATA_DIR = os.path.join(
    
    29
    -    os.path.dirname(os.path.realpath(__file__)),
    
    30
    -    "project",
    
    31
    -)
    
    32
    -
    
    33
    -
    
    34
    -@pytest.mark.datafiles(DATA_DIR)
    
    35
    -def test_source_bundle(cli, tmpdir, datafiles):
    
    36
    -    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    37
    -    element_name = 'source-bundle/source-bundle-hello.bst'
    
    38
    -    normal_name = 'source-bundle-source-bundle-hello'
    
    39
    -
    
    40
    -    # Verify that we can correctly produce a source-bundle
    
    41
    -    args = ['source-bundle', element_name, '--directory', str(tmpdir)]
    
    42
    -    result = cli.run(project=project_path, args=args)
    
    43
    -    result.assert_success()
    
    44
    -
    
    45
    -    # Verify that the source-bundle contains our sources and a build script
    
    46
    -    with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
    
    47
    -        assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
    
    48
    -        assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

  • tests/frontend/source_checkout.py
    1 1
     import os
    
    2 2
     import pytest
    
    3 3
     import tarfile
    
    4
    +from pathlib import Path
    
    4 5
     
    
    5 6
     from tests.testutils import cli
    
    6 7
     
    
    ... ... @@ -40,6 +41,22 @@ def test_source_checkout(datafiles, cli):
    40 41
         assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    41 42
     
    
    42 43
     
    
    44
    +@pytest.mark.datafiles(DATA_DIR)
    
    45
    +@pytest.mark.parametrize('force_flag', ['--force', '-f'])
    
    46
    +def test_source_checkout_force(datafiles, cli, force_flag):
    
    47
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    48
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    49
    +    target = 'checkout-deps.bst'
    
    50
    +
    
    51
    +    os.makedirs(os.path.join(checkout, 'some-thing'))
    
    52
    +    # Path(os.path.join(checkout, 'some-file')).touch()
    
    53
    +
    
    54
    +    result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
    
    55
    +    result.assert_success()
    
    56
    +
    
    57
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    58
    +
    
    59
    +
    
    43 60
     @pytest.mark.datafiles(DATA_DIR)
    
    44 61
     def test_source_checkout_tar(datafiles, cli):
    
    45 62
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    ... ... @@ -137,3 +154,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
    137 154
             assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
    
    138 155
         else:
    
    139 156
             result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
    
    157
    +
    
    158
    +
    
    159
    +@pytest.mark.datafiles(DATA_DIR)
    
    160
    +def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
    
    161
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    162
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    163
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    164
    +    checkout = os.path.join(str(tmpdir), 'source-checkout')
    
    165
    +
    
    166
    +    args = ['source-checkout', '--include-build-scripts', element_name, checkout]
    
    167
    +    result = cli.run(project=project_path, args=args)
    
    168
    +    result.assert_success()
    
    169
    +
    
    170
    +    # There sould be a script for each element (just one in this case) and a top level build script
    
    171
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    172
    +    for script in expected_scripts:
    
    173
    +        assert script in os.listdir(checkout)
    
    174
    +
    
    175
    +
    
    176
    +@pytest.mark.datafiles(DATA_DIR)
    
    177
    +def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
    
    178
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    179
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    180
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    181
    +    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
    
    182
    +
    
    183
    +    args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
    
    184
    +    result = cli.run(project=project_path, args=args)
    
    185
    +    result.assert_success()
    
    186
    +
    
    187
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    188
    +
    
    189
    +    with tarfile.open(tar_file, 'r') as tf:
    
    190
    +        for script in expected_scripts:
    
    191
    +            assert script in tf.getnames()



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]