[Notes] [Git][BuildStream/buildstream][issue-638-validate-all-files] 11 commits: Add --tar option to source-checkout command



Title: GitLab

Phillip Smyth pushed to branch issue-638-validate-all-files at BuildStream / buildstream

Commits:

19 changed files:

Changes:

  • NEWS
    ... ... @@ -2,10 +2,20 @@
    2 2
     buildstream 1.3.1
    
    3 3
     =================
    
    4 4
     
    
    5
    +  o BREAKING CHANGE: The bst source-bundle command has been removed. The
    
    6
    +    functionality it provided has been replaced by the `--include-build-scripts`
    
    7
    +    option of the `bst source-checkout` command. To produce a tarball containing
    
    8
    +    an element's sources and generated build scripts you can do the command
    
    9
    +    `bst source-checkout --include-build-scripts --tar foo.bst some-file.tar`
    
    10
    +
    
    5 11
       o BREAKING CHANGE: Default strip-commands have been removed as they are too
    
    6 12
         specific. Recommendation if you are building in Linux is to use the
    
    7 13
         ones being used in freedesktop-sdk project, for example
    
    8 14
     
    
    15
    +  o Running `bst show` without elements specified will now attempt to show
    
    16
    +    the default element defined in the projcet configuration.
    
    17
    +    If no default element is defined, all elements in the project will be shown
    
    18
    +
    
    9 19
       o All elements must now be suffixed with `.bst`
    
    10 20
         Attempting to use an element that does not have the `.bst` extension,
    
    11 21
         will result in a warning.
    

  • buildstream/_frontend/cli.py
    ... ... @@ -526,6 +526,11 @@ def push(app, elements, deps, remote):
    526 526
     def show(app, elements, deps, except_, order, format_):
    
    527 527
         """Show elements in the pipeline
    
    528 528
     
    
    529
    +    Declaring no elements with result in showing a default element if one is declared in the project configuration.
    
    530
    +
    
    531
    +    If no default is declared, all elements in the project will be shown
    
    532
    +
    
    533
    +
    
    529 534
         By default this will show all of the dependencies of the
    
    530 535
         specified target element.
    
    531 536
     
    
    ... ... @@ -570,11 +575,14 @@ def show(app, elements, deps, except_, order, format_):
    570 575
             bst show target.bst --format \\
    
    571 576
                 $'---------- %{name} ----------\\n%{vars}'
    
    572 577
         """
    
    578
    +
    
    573 579
         with app.initialized():
    
    574 580
             if not elements:
    
    575 581
                 guessed_target = app.context.guess_element()
    
    576 582
                 if guessed_target:
    
    577 583
                     elements = (guessed_target,)
    
    584
    +            else:
    
    585
    +                elements = app.project.get_default_elements()
    
    578 586
     
    
    579 587
             dependencies = app.stream.load_selection(elements,
    
    580 588
                                                      selection=deps,
    
    ... ... @@ -725,6 +733,8 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    725 733
     #                  Source Checkout Command                      #
    
    726 734
     ##################################################################
    
    727 735
     @cli.command(name='source-checkout', short_help='Checkout sources for an element')
    
    736
    +@click.option('--force', '-f', default=False, is_flag=True,
    
    737
    +              help="Allow files to be overwritten")
    
    728 738
     @click.option('--except', 'except_', multiple=True,
    
    729 739
                   type=click.Path(readable=False),
    
    730 740
                   help="Except certain dependencies")
    
    ... ... @@ -733,11 +743,15 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    733 743
                   help='The dependencies whose sources to checkout (default: none)')
    
    734 744
     @click.option('--fetch', 'fetch_', default=False, is_flag=True,
    
    735 745
                   help='Fetch elements if they are not fetched')
    
    736
    -@click.argument('element', required=False,
    
    737
    -                type=click.Path(readable=False))
    
    746
    +@click.option('--tar', 'tar', default=False, is_flag=True,
    
    747
    +              help='Create a tarball from the element\'s sources instead of a '
    
    748
    +                   'file tree.')
    
    749
    +@click.option('--include-build-scripts', 'build_scripts', is_flag=True)
    
    750
    +@click.argument('element', required=False, type=click.Path(readable=False))
    
    738 751
     @click.argument('location', type=click.Path(), required=False)
    
    739 752
     @click.pass_obj
    
    740
    -def source_checkout(app, element, location, deps, fetch_, except_):
    
    753
    +def source_checkout(app, element, location, force, deps, fetch_, except_,
    
    754
    +                    tar, build_scripts):
    
    741 755
         """Checkout sources of an element to the specified location
    
    742 756
         """
    
    743 757
         if not element and not location:
    
    ... ... @@ -757,9 +771,12 @@ def source_checkout(app, element, location, deps, fetch_, except_):
    757 771
     
    
    758 772
             app.stream.source_checkout(element,
    
    759 773
                                        location=location,
    
    774
    +                                   force=force,
    
    760 775
                                        deps=deps,
    
    761 776
                                        fetch=fetch_,
    
    762
    -                                   except_targets=except_)
    
    777
    +                                   except_targets=except_,
    
    778
    +                                   tar=tar,
    
    779
    +                                   include_build_scripts=build_scripts)
    
    763 780
     
    
    764 781
     
    
    765 782
     ##################################################################
    
    ... ... @@ -906,34 +923,3 @@ def workspace_list(app):
    906 923
     
    
    907 924
         with app.initialized():
    
    908 925
             app.stream.workspace_list()
    909
    -
    
    910
    -
    
    911
    -##################################################################
    
    912
    -#                     Source Bundle Command                      #
    
    913
    -##################################################################
    
    914
    -@cli.command(name="source-bundle", short_help="Produce a build bundle to be manually executed")
    
    915
    -@click.option('--except', 'except_', multiple=True,
    
    916
    -              type=click.Path(readable=False),
    
    917
    -              help="Elements to except from the tarball")
    
    918
    -@click.option('--compression', default='gz',
    
    919
    -              type=click.Choice(['none', 'gz', 'bz2', 'xz']),
    
    920
    -              help="Compress the tar file using the given algorithm.")
    
    921
    -@click.option('--track', 'track_', default=False, is_flag=True,
    
    922
    -              help="Track new source references before bundling")
    
    923
    -@click.option('--force', '-f', default=False, is_flag=True,
    
    924
    -              help="Overwrite an existing tarball")
    
    925
    -@click.option('--directory', default=os.getcwd(),
    
    926
    -              help="The directory to write the tarball to")
    
    927
    -@click.argument('element',
    
    928
    -                type=click.Path(readable=False))
    
    929
    -@click.pass_obj
    
    930
    -def source_bundle(app, element, force, directory,
    
    931
    -                  track_, compression, except_):
    
    932
    -    """Produce a source bundle to be manually executed
    
    933
    -    """
    
    934
    -    with app.initialized():
    
    935
    -        app.stream.source_bundle(element, directory,
    
    936
    -                                 track_first=track_,
    
    937
    -                                 force=force,
    
    938
    -                                 compression=compression,
    
    939
    -                                 except_targets=except_)

  • buildstream/_project.py
    ... ... @@ -228,7 +228,7 @@ class Project():
    228 228
                 'element-path', 'variables',
    
    229 229
                 'environment', 'environment-nocache',
    
    230 230
                 'split-rules', 'elements', 'plugins',
    
    231
    -            'aliases', 'name',
    
    231
    +            'aliases', 'name', 'defaults',
    
    232 232
                 'artifacts', 'options',
    
    233 233
                 'fail-on-overlap', 'shell', 'fatal-warnings',
    
    234 234
                 'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
    
    ... ... @@ -391,6 +391,36 @@ class Project():
    391 391
             # Reset the element loader state
    
    392 392
             Element._reset_load_state()
    
    393 393
     
    
    394
    +
    
    395
    +    # get_default_elements()
    
    396
    +    #
    
    397
    +    # This function is used to gather either:
    
    398
    +    # The project default element (if defined in project.conf)
    
    399
    +    # or
    
    400
    +    # All elements in the project
    
    401
    +    #
    
    402
    +    def get_default_elements(self):
    
    403
    +        output = []
    
    404
    +
    
    405
    +        # The project is not required to have an element-path
    
    406
    +        element_directory = self._project_conf.get('element-path')
    
    407
    +
    
    408
    +        # The project may have a default element defined
    
    409
    +        default_element = self._project_conf.get("defaults", {}).get("target-element", None)
    
    410
    +
    
    411
    +        if default_element:
    
    412
    +            return (default_element,)
    
    413
    +
    
    414
    +        directory = os.path.join(self.directory, element_directory)
    
    415
    +        for root, _, files in os.walk(directory):
    
    416
    +            for file in files:
    
    417
    +                if file.endswith(".bst"):
    
    418
    +                    rel_dir = os.path.relpath(root, directory)
    
    419
    +                    rel_file = os.path.join(rel_dir, file).lstrip("./")
    
    420
    +                    output.append(rel_file)
    
    421
    +        return tuple(output)
    
    422
    +
    
    423
    +
    
    394 424
         # _load():
    
    395 425
         #
    
    396 426
         # Loads the project configuration file in the project
    

  • buildstream/_stream.py
    ... ... @@ -25,8 +25,8 @@ import stat
    25 25
     import shlex
    
    26 26
     import shutil
    
    27 27
     import tarfile
    
    28
    -from contextlib import contextmanager
    
    29
    -from tempfile import TemporaryDirectory
    
    28
    +import tempfile
    
    29
    +from contextlib import contextmanager, suppress
    
    30 30
     
    
    31 31
     from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
    
    32 32
     from ._message import Message, MessageType
    
    ... ... @@ -449,11 +449,14 @@ class Stream():
    449 449
         #
    
    450 450
         def source_checkout(self, target, *,
    
    451 451
                             location=None,
    
    452
    +                        force=False,
    
    452 453
                             deps='none',
    
    453 454
                             fetch=False,
    
    454
    -                        except_targets=()):
    
    455
    +                        except_targets=(),
    
    456
    +                        tar=False,
    
    457
    +                        include_build_scripts=False):
    
    455 458
     
    
    456
    -        self._check_location_writable(location)
    
    459
    +        self._check_location_writable(location, force=force, tar=tar)
    
    457 460
     
    
    458 461
             elements, _ = self._load((target,), (),
    
    459 462
                                      selection=deps,
    
    ... ... @@ -467,7 +470,8 @@ class Stream():
    467 470
     
    
    468 471
             # Stage all sources determined by scope
    
    469 472
             try:
    
    470
    -            self._write_element_sources(location, elements)
    
    473
    +            self._source_checkout(elements, location, force, deps,
    
    474
    +                                  fetch, tar, include_build_scripts)
    
    471 475
             except BstError as e:
    
    472 476
                 raise StreamError("Error while writing sources"
    
    473 477
                                   ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    ... ... @@ -728,87 +732,6 @@ class Stream():
    728 732
                 'workspaces': workspaces
    
    729 733
             })
    
    730 734
     
    
    731
    -    # source_bundle()
    
    732
    -    #
    
    733
    -    # Create a host buildable tarball bundle for the given target.
    
    734
    -    #
    
    735
    -    # Args:
    
    736
    -    #    target (str): The target element to bundle
    
    737
    -    #    directory (str): The directory to output the tarball
    
    738
    -    #    track_first (bool): Track new source references before bundling
    
    739
    -    #    compression (str): The compression type to use
    
    740
    -    #    force (bool): Overwrite an existing tarball
    
    741
    -    #
    
    742
    -    def source_bundle(self, target, directory, *,
    
    743
    -                      track_first=False,
    
    744
    -                      force=False,
    
    745
    -                      compression="gz",
    
    746
    -                      except_targets=()):
    
    747
    -
    
    748
    -        if track_first:
    
    749
    -            track_targets = (target,)
    
    750
    -        else:
    
    751
    -            track_targets = ()
    
    752
    -
    
    753
    -        elements, track_elements = self._load((target,), track_targets,
    
    754
    -                                              selection=PipelineSelection.ALL,
    
    755
    -                                              except_targets=except_targets,
    
    756
    -                                              track_selection=PipelineSelection.ALL,
    
    757
    -                                              fetch_subprojects=True)
    
    758
    -
    
    759
    -        # source-bundle only supports one target
    
    760
    -        target = self.targets[0]
    
    761
    -
    
    762
    -        self._message(MessageType.INFO, "Bundling sources for target {}".format(target.name))
    
    763
    -
    
    764
    -        # Find the correct filename for the compression algorithm
    
    765
    -        tar_location = os.path.join(directory, target.normal_name + ".tar")
    
    766
    -        if compression != "none":
    
    767
    -            tar_location += "." + compression
    
    768
    -
    
    769
    -        # Attempt writing a file to generate a good error message
    
    770
    -        # early
    
    771
    -        #
    
    772
    -        # FIXME: A bit hackish
    
    773
    -        try:
    
    774
    -            open(tar_location, mode="x")
    
    775
    -            os.remove(tar_location)
    
    776
    -        except IOError as e:
    
    777
    -            raise StreamError("Cannot write to {0}: {1}"
    
    778
    -                              .format(tar_location, e)) from e
    
    779
    -
    
    780
    -        # Fetch and possibly track first
    
    781
    -        #
    
    782
    -        self._fetch(elements, track_elements=track_elements)
    
    783
    -
    
    784
    -        # We don't use the scheduler for this as it is almost entirely IO
    
    785
    -        # bound.
    
    786
    -
    
    787
    -        # Create a temporary directory to build the source tree in
    
    788
    -        builddir = self._context.builddir
    
    789
    -        os.makedirs(builddir, exist_ok=True)
    
    790
    -        prefix = "{}-".format(target.normal_name)
    
    791
    -
    
    792
    -        with TemporaryDirectory(prefix=prefix, dir=builddir) as tempdir:
    
    793
    -            source_directory = os.path.join(tempdir, 'source')
    
    794
    -            try:
    
    795
    -                os.makedirs(source_directory)
    
    796
    -            except OSError as e:
    
    797
    -                raise StreamError("Failed to create directory: {}"
    
    798
    -                                  .format(e)) from e
    
    799
    -
    
    800
    -            # Any elements that don't implement _write_script
    
    801
    -            # should not be included in the later stages.
    
    802
    -            elements = [
    
    803
    -                element for element in elements
    
    804
    -                if self._write_element_script(source_directory, element)
    
    805
    -            ]
    
    806
    -
    
    807
    -            self._write_element_sources(os.path.join(tempdir, "source"), elements)
    
    808
    -            self._write_build_script(tempdir, elements)
    
    809
    -            self._collect_sources(tempdir, tar_location,
    
    810
    -                                  target.normal_name, compression)
    
    811
    -
    
    812 735
         # redirect_element_names()
    
    813 736
         #
    
    814 737
         # Takes a list of element names and returns a list where elements have been
    
    ... ... @@ -1189,6 +1112,54 @@ class Stream():
    1189 1112
     
    
    1190 1113
             sandbox_vroot.export_files(directory, can_link=True, can_destroy=True)
    
    1191 1114
     
    
    1115
    +    # Helper function for source_checkout()
    
    1116
    +    def _source_checkout(self, elements,
    
    1117
    +                         location=None,
    
    1118
    +                         force=False,
    
    1119
    +                         deps='none',
    
    1120
    +                         fetch=False,
    
    1121
    +                         tar=False,
    
    1122
    +                         include_build_scripts=False):
    
    1123
    +        location = os.path.abspath(location)
    
    1124
    +        location_parent = os.path.abspath(os.path.join(location, ".."))
    
    1125
    +
    
    1126
    +        # Stage all our sources in a temporary directory. The this
    
    1127
    +        # directory can be used to either construct a tarball or moved
    
    1128
    +        # to the final desired location.
    
    1129
    +        temp_source_dir = tempfile.TemporaryDirectory(dir=location_parent)
    
    1130
    +        try:
    
    1131
    +            self._write_element_sources(temp_source_dir.name, elements)
    
    1132
    +            if include_build_scripts:
    
    1133
    +                self._write_build_scripts(temp_source_dir.name, elements)
    
    1134
    +            if tar:
    
    1135
    +                self._create_tarball(temp_source_dir.name, location)
    
    1136
    +            else:
    
    1137
    +                self._move_directory(temp_source_dir.name, location, force)
    
    1138
    +        except OSError as e:
    
    1139
    +            raise StreamError("Failed to checkout sources to {}: {}"
    
    1140
    +                              .format(location, e)) from e
    
    1141
    +        finally:
    
    1142
    +            with suppress(FileNotFoundError):
    
    1143
    +                temp_source_dir.cleanup()
    
    1144
    +
    
    1145
    +    # Move a directory src to dest. This will work across devices and
    
    1146
    +    # may optionaly overwrite existing files.
    
    1147
    +    def _move_directory(self, src, dest, force=False):
    
    1148
    +        def is_empty_dir(path):
    
    1149
    +            return os.path.isdir(dest) and not os.listdir(dest)
    
    1150
    +
    
    1151
    +        try:
    
    1152
    +            os.rename(src, dest)
    
    1153
    +            return
    
    1154
    +        except OSError:
    
    1155
    +            pass
    
    1156
    +
    
    1157
    +        if force or is_empty_dir(dest):
    
    1158
    +            try:
    
    1159
    +                utils.link_files(src, dest)
    
    1160
    +            except utils.UtilError as e:
    
    1161
    +                raise StreamError("Failed to move directory: {}".format(e)) from e
    
    1162
    +
    
    1192 1163
         # Write the element build script to the given directory
    
    1193 1164
         def _write_element_script(self, directory, element):
    
    1194 1165
             try:
    
    ... ... @@ -1205,8 +1176,28 @@ class Stream():
    1205 1176
                     os.makedirs(element_source_dir)
    
    1206 1177
                     element._stage_sources_at(element_source_dir, mount_workspaces=False)
    
    1207 1178
     
    
    1179
    +    # Create a tarball from the content of directory
    
    1180
    +    def _create_tarball(self, directory, tar_name):
    
    1181
    +        try:
    
    1182
    +            with utils.save_file_atomic(tar_name, mode='wb') as f:
    
    1183
    +                # This TarFile does not need to be explicitly closed
    
    1184
    +                # as the underlying file object will be closed be the
    
    1185
    +                # save_file_atomic contect manager
    
    1186
    +                tarball = tarfile.open(fileobj=f, mode='w')
    
    1187
    +                for item in os.listdir(str(directory)):
    
    1188
    +                    file_to_add = os.path.join(directory, item)
    
    1189
    +                    tarball.add(file_to_add, arcname=item)
    
    1190
    +        except OSError as e:
    
    1191
    +            raise StreamError("Failed to create tar archive: {}".format(e)) from e
    
    1192
    +
    
    1193
    +    # Write all the build_scripts for elements in the directory location
    
    1194
    +    def _write_build_scripts(self, location, elements):
    
    1195
    +        for element in elements:
    
    1196
    +            self._write_element_script(location, element)
    
    1197
    +        self._write_master_build_script(location, elements)
    
    1198
    +
    
    1208 1199
         # Write a master build script to the sandbox
    
    1209
    -    def _write_build_script(self, directory, elements):
    
    1200
    +    def _write_master_build_script(self, directory, elements):
    
    1210 1201
     
    
    1211 1202
             module_string = ""
    
    1212 1203
             for element in elements:
    

  • buildstream/data/projectconfig.yaml
    ... ... @@ -167,3 +167,10 @@ shell:
    167 167
       # Command to run when `bst shell` does not provide a command
    
    168 168
       #
    
    169 169
       command: [ 'sh', '-i' ]
    
    170
    +
    
    171
    +# Default Targets
    
    172
    +#
    
    173
    +defaults:
    
    174
    +
    
    175
    +  # Set a Default element to build when none are defined
    
    176
    +  target-element: None

  • doc/source/using_commands.rst
    ... ... @@ -86,13 +86,6 @@ project's main directory.
    86 86
     
    
    87 87
     ----
    
    88 88
     
    
    89
    -.. _invoking_source_bundle:
    
    90
    -
    
    91
    -.. click:: buildstream._frontend.cli:source_bundle
    
    92
    -   :prog: bst source bundle
    
    93
    -
    
    94
    -----
    
    95
    -
    
    96 89
     .. _invoking_workspace:
    
    97 90
     
    
    98 91
     .. click:: buildstream._frontend.cli:workspace
    

  • tests/completions/completions.py
    ... ... @@ -16,7 +16,6 @@ MAIN_COMMANDS = [
    16 16
         'shell ',
    
    17 17
         'show ',
    
    18 18
         'source-checkout ',
    
    19
    -    'source-bundle ',
    
    20 19
         'track ',
    
    21 20
         'workspace '
    
    22 21
     ]
    

  • tests/frontend/help.py
    ... ... @@ -25,7 +25,6 @@ def test_help_main(cli):
    25 25
         ('push'),
    
    26 26
         ('shell'),
    
    27 27
         ('show'),
    
    28
    -    ('source-bundle'),
    
    29 28
         ('track'),
    
    30 29
         ('workspace')
    
    31 30
     ])
    

  • tests/frontend/project_default/elements/target.bst
    1
    +kind: stack
    
    2
    +description: |
    
    3
    +
    
    4
    +  Main stack target for the bst build test

  • tests/frontend/project_default/elements/target2.bst
    1
    +kind: stack
    
    2
    +description: |
    
    3
    +
    
    4
    +  Main stack target for the bst build test

  • tests/frontend/project_default/project.conf
    1
    +# Project config for frontend build test
    
    2
    +name: test
    
    3
    +
    
    4
    +element-path: elements
    
    5
    +
    
    6
    +fatal-warnings:
    
    7
    +- bad-element-suffix
    
    8
    +
    
    9
    +defaults:
    
    10
    + target-element: target2.bst

  • tests/frontend/project_fail/elements/compose-all.bst
    1
    +kind: compose
    
    2
    +
    
    3
    +depends:
    
    4
    +- fileNAME: import-dev.bst
    
    5
    +  type: build
    
    6
    +
    
    7
    +config:
    
    8
    +  # Dont try running the sandbox, we dont have a
    
    9
    +  # runtime to run anything in this context.
    
    10
    +  integrate: False

  • tests/frontend/project_fail/elements/import-dev.bst
    1
    +kind: import
    
    2
    +sources:
    
    3
    +- kind: local
    
    4
    +  path: files/dev-files

  • tests/frontend/project_fail/elements/target.bst
    1
    +kind: stack
    
    2
    +description: |
    
    3
    +
    
    4
    +  Main stack target for the bst build test
    
    5
    +
    
    6
    +depends:
    
    7
    +- compose-all.bst

  • tests/frontend/project_fail/files/dev-files/usr/include/pony.h
    1
    +#ifndef __PONY_H__
    
    2
    +#define __PONY_H__
    
    3
    +
    
    4
    +#define PONY_BEGIN "Once upon a time, there was a pony."
    
    5
    +#define PONY_END "And they lived happily ever after, the end."
    
    6
    +
    
    7
    +#define MAKE_PONY(story)  \
    
    8
    +  PONY_BEGIN \
    
    9
    +  story \
    
    10
    +  PONY_END
    
    11
    +
    
    12
    +#endif /* __PONY_H__ */

  • tests/frontend/project_fail/project.conf
    1
    +# Project config for frontend build test
    
    2
    +name: test
    
    3
    +
    
    4
    +element-path: elements

  • tests/frontend/show.py
    ... ... @@ -46,6 +46,27 @@ def test_show_invalid_element_path(cli, datafiles):
    46 46
             'show',
    
    47 47
             "foo.bst"])
    
    48 48
     
    
    49
    +
    
    50
    +@pytest.mark.datafiles(DATA_DIR + "_default")
    
    51
    +def test_show_default(cli, datafiles):
    
    52
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    53
    +    result = cli.run(project=project, cwd=project, silent=True, args=[
    
    54
    +        'show'])
    
    55
    +
    
    56
    +    result.assert_success()
    
    57
    +
    
    58
    +    # Get the result output of "[state sha element]" and turn into a list 
    
    59
    +    results = result.output.strip().splitlines().split(" ")
    
    60
    +    expected = 'target2.bst'
    
    61
    +    assert results[2] == expected
    
    62
    +
    
    63
    +
    
    64
    +@pytest.mark.datafiles(DATA_DIR + "_fail")
    
    65
    +def test_show_fail(cli, datafiles):
    
    66
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    67
    +    result = cli.run(project=project, cwd=project, silent=True, args=[
    
    68
    +        'show'])
    
    69
    +
    
    49 70
         result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
    
    50 71
     
    
    51 72
     
    

  • tests/frontend/source_bundle.py deleted
    1
    -#
    
    2
    -#  Copyright (C) 2018 Bloomberg Finance LP
    
    3
    -#
    
    4
    -#  This program is free software; you can redistribute it and/or
    
    5
    -#  modify it under the terms of the GNU Lesser General Public
    
    6
    -#  License as published by the Free Software Foundation; either
    
    7
    -#  version 2 of the License, or (at your option) any later version.
    
    8
    -#
    
    9
    -#  This library is distributed in the hope that it will be useful,
    
    10
    -#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    -#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    
    12
    -#  Lesser General Public License for more details.
    
    13
    -#
    
    14
    -#  You should have received a copy of the GNU Lesser General Public
    
    15
    -#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    -#
    
    17
    -#  Authors: Chandan Singh <csingh43 bloomberg net>
    
    18
    -#
    
    19
    -
    
    20
    -import os
    
    21
    -import tarfile
    
    22
    -
    
    23
    -import pytest
    
    24
    -
    
    25
    -from tests.testutils import cli
    
    26
    -
    
    27
    -# Project directory
    
    28
    -DATA_DIR = os.path.join(
    
    29
    -    os.path.dirname(os.path.realpath(__file__)),
    
    30
    -    "project",
    
    31
    -)
    
    32
    -
    
    33
    -
    
    34
    -@pytest.mark.datafiles(DATA_DIR)
    
    35
    -def test_source_bundle(cli, tmpdir, datafiles):
    
    36
    -    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    37
    -    element_name = 'source-bundle/source-bundle-hello.bst'
    
    38
    -    normal_name = 'source-bundle-source-bundle-hello'
    
    39
    -
    
    40
    -    # Verify that we can correctly produce a source-bundle
    
    41
    -    args = ['source-bundle', element_name, '--directory', str(tmpdir)]
    
    42
    -    result = cli.run(project=project_path, args=args)
    
    43
    -    result.assert_success()
    
    44
    -
    
    45
    -    # Verify that the source-bundle contains our sources and a build script
    
    46
    -    with tarfile.open(os.path.join(str(tmpdir), '{}.tar.gz'.format(normal_name))) as bundle:
    
    47
    -        assert os.path.join(normal_name, 'source', normal_name, 'llamas.txt') in bundle.getnames()
    
    48
    -        assert os.path.join(normal_name, 'build.sh') in bundle.getnames()

  • tests/frontend/source_checkout.py
    1 1
     import os
    
    2 2
     import pytest
    
    3
    +import tarfile
    
    4
    +from pathlib import Path
    
    3 5
     
    
    4 6
     from tests.testutils import cli
    
    5 7
     
    
    ... ... @@ -55,6 +57,39 @@ def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_e
    55 57
         assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    56 58
     
    
    57 59
     
    
    60
    +@pytest.mark.datafiles(DATA_DIR)
    
    61
    +@pytest.mark.parametrize('force_flag', ['--force', '-f'])
    
    62
    +def test_source_checkout_force(datafiles, cli, force_flag):
    
    63
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    64
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    65
    +    target = 'checkout-deps.bst'
    
    66
    +
    
    67
    +    os.makedirs(os.path.join(checkout, 'some-thing'))
    
    68
    +    # Path(os.path.join(checkout, 'some-file')).touch()
    
    69
    +
    
    70
    +    result = cli.run(project=project, args=['source-checkout', force_flag, target, '--deps', 'none', checkout])
    
    71
    +    result.assert_success()
    
    72
    +
    
    73
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    74
    +
    
    75
    +
    
    76
    +@pytest.mark.datafiles(DATA_DIR)
    
    77
    +def test_source_checkout_tar(datafiles, cli):
    
    78
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    79
    +    checkout = os.path.join(cli.directory, 'source-checkout.tar')
    
    80
    +    target = 'checkout-deps.bst'
    
    81
    +
    
    82
    +    result = cli.run(project=project, args=['source-checkout', '--tar', target, '--deps', 'none', checkout])
    
    83
    +    result.assert_success()
    
    84
    +
    
    85
    +    assert os.path.exists(checkout)
    
    86
    +    with tarfile.open(checkout) as tf:
    
    87
    +        expected_content = os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config')
    
    88
    +        tar_members = [f.name for f in tf]
    
    89
    +        for member in tar_members:
    
    90
    +            assert member in expected_content
    
    91
    +
    
    92
    +
    
    58 93
     @pytest.mark.datafiles(DATA_DIR)
    
    59 94
     @pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
    
    60 95
     def test_source_checkout_deps(datafiles, cli, deps):
    
    ... ... @@ -135,3 +170,38 @@ def test_source_checkout_fetch(datafiles, cli, fetch):
    135 170
             assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
    
    136 171
         else:
    
    137 172
             result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')
    
    173
    +
    
    174
    +
    
    175
    +@pytest.mark.datafiles(DATA_DIR)
    
    176
    +def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
    
    177
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    178
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    179
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    180
    +    checkout = os.path.join(str(tmpdir), 'source-checkout')
    
    181
    +
    
    182
    +    args = ['source-checkout', '--include-build-scripts', element_name, checkout]
    
    183
    +    result = cli.run(project=project_path, args=args)
    
    184
    +    result.assert_success()
    
    185
    +
    
    186
    +    # There sould be a script for each element (just one in this case) and a top level build script
    
    187
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    188
    +    for script in expected_scripts:
    
    189
    +        assert script in os.listdir(checkout)
    
    190
    +
    
    191
    +
    
    192
    +@pytest.mark.datafiles(DATA_DIR)
    
    193
    +def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
    
    194
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    195
    +    element_name = 'source-bundle/source-bundle-hello.bst'
    
    196
    +    normal_name = 'source-bundle-source-bundle-hello'
    
    197
    +    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
    
    198
    +
    
    199
    +    args = ['source-checkout', '--include-build-scripts', '--tar', element_name, tar_file]
    
    200
    +    result = cli.run(project=project_path, args=args)
    
    201
    +    result.assert_success()
    
    202
    +
    
    203
    +    expected_scripts = ['build.sh', 'build-' + normal_name]
    
    204
    +
    
    205
    +    with tarfile.open(tar_file, 'r') as tf:
    
    206
    +        for script in expected_scripts:
    
    207
    +            assert script in tf.getnames()



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]