[Notes] [Git][BuildStream/buildstream][chandan/source-checkout] Add `bst source-checkout` command



Title: GitLab

Chandan Singh pushed to branch chandan/source-checkout at BuildStream / buildstream

Commits:

7 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -662,6 +662,33 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    662 662
                                 tar=tar)
    
    663 663
     
    
    664 664
     
    
    665
    +##################################################################
    
    666
    +#                  Source Checkout Command                      #
    
    667
    +##################################################################
    
    668
    +@cli.command(name='source-checkout', short_help='Checkout sources for an element')
    
    669
    +@click.option('--except', 'except_', multiple=True,
    
    670
    +              type=click.Path(readable=False),
    
    671
    +              help="Except certain dependencies")
    
    672
    +@click.option('--deps', '-d', default='none',
    
    673
    +              type=click.Choice(['build', 'none', 'run', 'all']),
    
    674
    +              help='The dependencies whose sources to checkout (default: none)')
    
    675
    +@click.option('--fetch', 'fetch_', default=False, is_flag=True,
    
    676
    +              help='Fetch elements if they are not fetched')
    
    677
    +@click.argument('element',
    
    678
    +                type=click.Path(readable=False))
    
    679
    +@click.argument('location', type=click.Path())
    
    680
    +@click.pass_obj
    
    681
    +def source_checkout(app, element, location, deps, fetch_, except_):
    
    682
    +    """Checkout sources of an element to the specified location
    
    683
    +    """
    
    684
    +    with app.initialized():
    
    685
    +        app.stream.source_checkout(element,
    
    686
    +                                   location=location,
    
    687
    +                                   deps=deps,
    
    688
    +                                   fetch=fetch_,
    
    689
    +                                   except_targets=except_)
    
    690
    +
    
    691
    +
    
    665 692
     ##################################################################
    
    666 693
     #                      Workspace Command                         #
    
    667 694
     ##################################################################
    

  • buildstream/_pipeline.py
    ... ... @@ -383,6 +383,33 @@ class Pipeline():
    383 383
                     detail += "  " + element._get_full_name() + "\n"
    
    384 384
                 raise PipelineError("Inconsistent pipeline", detail=detail, reason="inconsistent-pipeline-workspaced")
    
    385 385
     
    
    386
    +    # assert_sources_cached()
    
    387
    +    #
    
    388
    +    # Asserts that sources for the given list of elements are cached.
    
    389
    +    #
    
    390
    +    # Args:
    
    391
    +    #    elements (list): The list of elements
    
    392
    +    #
    
    393
    +    def assert_sources_cached(self, elements):
    
    394
    +        uncached = []
    
    395
    +        with self._context.timed_activity("Checking sources"):
    
    396
    +            for element in elements:
    
    397
    +                if element._get_consistency() != Consistency.CACHED:
    
    398
    +                    uncached.append(element)
    
    399
    +
    
    400
    +        if uncached:
    
    401
    +            detail = "Sources are not cached for the following elements:\n\n"
    
    402
    +            for element in uncached:
    
    403
    +                detail += "  Following sources for element: {} are not cached:\n".format(element._get_full_name())
    
    404
    +                for source in element.sources():
    
    405
    +                    if source._get_consistency() != Consistency.CACHED:
    
    406
    +                        detail += "    {}\n".format(source)
    
    407
    +                detail += '\n'
    
    408
    +            detail += "Try fetching these elements first with `bst fetch`,\n" + \
    
    409
    +                      "or run this command with `--fetch` option\n"
    
    410
    +
    
    411
    +            raise PipelineError("Uncached sources", detail=detail, reason="uncached-sources")
    
    412
    +
    
    386 413
         #############################################################
    
    387 414
         #                     Private Methods                       #
    
    388 415
         #############################################################
    

  • buildstream/_stream.py
    ... ... @@ -379,27 +379,7 @@ class Stream():
    379 379
             elements, _ = self._load((target,), (), fetch_subprojects=True)
    
    380 380
             target = elements[0]
    
    381 381
     
    
    382
    -        if not tar:
    
    383
    -            try:
    
    384
    -                os.makedirs(location, exist_ok=True)
    
    385
    -            except OSError as e:
    
    386
    -                raise StreamError("Failed to create checkout directory: '{}'"
    
    387
    -                                  .format(e)) from e
    
    388
    -
    
    389
    -        if not tar:
    
    390
    -            if not os.access(location, os.W_OK):
    
    391
    -                raise StreamError("Checkout directory '{}' not writable"
    
    392
    -                                  .format(location))
    
    393
    -            if not force and os.listdir(location):
    
    394
    -                raise StreamError("Checkout directory '{}' not empty"
    
    395
    -                                  .format(location))
    
    396
    -        elif os.path.exists(location) and location != '-':
    
    397
    -            if not os.access(location, os.W_OK):
    
    398
    -                raise StreamError("Output file '{}' not writable"
    
    399
    -                                  .format(location))
    
    400
    -            if not force and os.path.exists(location):
    
    401
    -                raise StreamError("Output file '{}' already exists"
    
    402
    -                                  .format(location))
    
    382
    +        self._check_location_writable(location, force=force, tar=tar)
    
    403 383
     
    
    404 384
             # Stage deps into a temporary sandbox first
    
    405 385
             try:
    
    ... ... @@ -436,6 +416,42 @@ class Stream():
    436 416
                 raise StreamError("Error while staging dependencies into a sandbox"
    
    437 417
                                   ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    438 418
     
    
    419
    +    # source_checkout()
    
    420
    +    #
    
    421
    +    # Checkout sources of the target element to the specified location
    
    422
    +    #
    
    423
    +    # Args:
    
    424
    +    #    target (str): The target element whose sources to checkout
    
    425
    +    #    location (str): Location to checkout the sources to
    
    426
    +    #    deps (str): The dependencies to checkout
    
    427
    +    #    fetch (bool): Whether to fetch missing sources
    
    428
    +    #    except_targets (list): List of targets to except from staging
    
    429
    +    #
    
    430
    +    def source_checkout(self, target, *,
    
    431
    +                        location=None,
    
    432
    +                        deps='none',
    
    433
    +                        fetch=False,
    
    434
    +                        except_targets=()):
    
    435
    +
    
    436
    +        self._check_location_writable(location)
    
    437
    +
    
    438
    +        elements, _ = self._load((target,), (),
    
    439
    +                                 selection=deps,
    
    440
    +                                 except_targets=except_targets,
    
    441
    +                                 fetch_subprojects=True)
    
    442
    +
    
    443
    +        # Assert all sources are cached
    
    444
    +        if fetch:
    
    445
    +            self._fetch(elements)
    
    446
    +        self._pipeline.assert_sources_cached(elements)
    
    447
    +
    
    448
    +        # Stage all sources determined by scope
    
    449
    +        try:
    
    450
    +            self._write_element_sources(location, elements)
    
    451
    +        except BstError as e:
    
    452
    +            raise StreamError("Error while writing sources"
    
    453
    +                              ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    454
    +
    
    439 455
         # workspace_open
    
    440 456
         #
    
    441 457
         # Open a project workspace
    
    ... ... @@ -719,7 +735,7 @@ class Stream():
    719 735
                     if self._write_element_script(source_directory, element)
    
    720 736
                 ]
    
    721 737
     
    
    722
    -            self._write_element_sources(tempdir, elements)
    
    738
    +            self._write_element_sources(os.path.join(tempdir, "source"), elements)
    
    723 739
                 self._write_build_script(tempdir, elements)
    
    724 740
                 self._collect_sources(tempdir, tar_location,
    
    725 741
                                       target.normal_name, compression)
    
    ... ... @@ -1061,6 +1077,39 @@ class Stream():
    1061 1077
             self._enqueue_plan(fetch_plan)
    
    1062 1078
             self._run()
    
    1063 1079
     
    
    1080
    +    # _check_location_writable()
    
    1081
    +    #
    
    1082
    +    # Check if given location is writable.
    
    1083
    +    #
    
    1084
    +    # Args:
    
    1085
    +    #    location (str): Destination path
    
    1086
    +    #    force (bool): Allow files to be overwritten
    
    1087
    +    #    tar (bool): Whether destination is a tarball
    
    1088
    +    #
    
    1089
    +    # Raises:
    
    1090
    +    #    (StreamError): If the destination is not writable
    
    1091
    +    #
    
    1092
    +    def _check_location_writable(self, location, force=False, tar=False):
    
    1093
    +        if not tar:
    
    1094
    +            try:
    
    1095
    +                os.makedirs(location, exist_ok=True)
    
    1096
    +            except OSError as e:
    
    1097
    +                raise StreamError("Failed to create destination directory: '{}'"
    
    1098
    +                                  .format(e)) from e
    
    1099
    +            if not os.access(location, os.W_OK):
    
    1100
    +                raise StreamError("Destination directory '{}' not writable"
    
    1101
    +                                  .format(location))
    
    1102
    +            if not force and os.listdir(location):
    
    1103
    +                raise StreamError("Destination directory '{}' not empty"
    
    1104
    +                                  .format(location))
    
    1105
    +        elif os.path.exists(location) and location != '-':
    
    1106
    +            if not os.access(location, os.W_OK):
    
    1107
    +                raise StreamError("Output file '{}' not writable"
    
    1108
    +                                  .format(location))
    
    1109
    +            if not force and os.path.exists(location):
    
    1110
    +                raise StreamError("Output file '{}' already exists"
    
    1111
    +                                  .format(location))
    
    1112
    +
    
    1064 1113
         # Helper function for checkout()
    
    1065 1114
         #
    
    1066 1115
         def _checkout_hardlinks(self, sandbox_vroot, directory):
    
    ... ... @@ -1082,11 +1131,10 @@ class Stream():
    1082 1131
         # Write all source elements to the given directory
    
    1083 1132
         def _write_element_sources(self, directory, elements):
    
    1084 1133
             for element in elements:
    
    1085
    -            source_dir = os.path.join(directory, "source")
    
    1086
    -            element_source_dir = os.path.join(source_dir, element.normal_name)
    
    1087
    -            os.makedirs(element_source_dir)
    
    1088
    -
    
    1089
    -            element._stage_sources_at(element_source_dir)
    
    1134
    +            element_source_dir = self._get_element_dirname(directory, element)
    
    1135
    +            if list(element.sources()):
    
    1136
    +                os.makedirs(element_source_dir)
    
    1137
    +                element._stage_sources_at(element_source_dir)
    
    1090 1138
     
    
    1091 1139
         # Write a master build script to the sandbox
    
    1092 1140
         def _write_build_script(self, directory, elements):
    
    ... ... @@ -1115,3 +1163,25 @@ class Stream():
    1115 1163
     
    
    1116 1164
                 with tarfile.open(tar_name, permissions) as tar:
    
    1117 1165
                     tar.add(directory, arcname=element_name)
    
    1166
    +
    
    1167
    +    # _get_element_dirname()
    
    1168
    +    #
    
    1169
    +    # Get path to directory for an element based on its normal name.
    
    1170
    +    #
    
    1171
    +    # For cross-junction elements, the path will be prefixed with the name
    
    1172
    +    # of the junction element.
    
    1173
    +    #
    
    1174
    +    # Args:
    
    1175
    +    #    directory (str): path to base directory
    
    1176
    +    #    element (Element): the element
    
    1177
    +    #
    
    1178
    +    # Returns:
    
    1179
    +    #    (str): Path to directory for this element
    
    1180
    +    #
    
    1181
    +    def _get_element_dirname(self, directory, element):
    
    1182
    +        parts = [element.normal_name]
    
    1183
    +        while element._get_project() != self._project:
    
    1184
    +            element = element._get_project().junction
    
    1185
    +            parts.append(element.normal_name)
    
    1186
    +
    
    1187
    +        return os.path.join(directory, *reversed(parts))

  • tests/completions/completions.py
    ... ... @@ -15,6 +15,7 @@ MAIN_COMMANDS = [
    15 15
         'push ',
    
    16 16
         'shell ',
    
    17 17
         'show ',
    
    18
    +    'source-checkout ',
    
    18 19
         'source-bundle ',
    
    19 20
         'track ',
    
    20 21
         'workspace '
    

  • tests/frontend/project/elements/checkout-deps.bst
    1
    +kind: import
    
    2
    +description: It is important for this element to have both build and runtime dependencies
    
    3
    +sources:
    
    4
    +- kind: local
    
    5
    +  path: files/etc-files
    
    6
    +depends:
    
    7
    +- filename: import-dev.bst
    
    8
    +  type: build
    
    9
    +- filename: import-bin.bst
    
    10
    +  type: runtime

  • tests/frontend/project/files/etc-files/etc/buildstream/config
    1
    +config

  • tests/frontend/source_checkout.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from tests.testutils import cli
    
    5
    +
    
    6
    +from buildstream import utils, _yaml
    
    7
    +from buildstream._exceptions import ErrorDomain, LoadErrorReason
    
    8
    +
    
    9
    +# Project directory
    
    10
    +DATA_DIR = os.path.join(
    
    11
    +    os.path.dirname(os.path.realpath(__file__)),
    
    12
    +    'project',
    
    13
    +)
    
    14
    +
    
    15
    +
    
    16
    +def generate_remote_import_element(input_path, output_path):
    
    17
    +    return {
    
    18
    +        'kind': 'import',
    
    19
    +        'sources': [
    
    20
    +            {
    
    21
    +                'kind': 'remote',
    
    22
    +                'url': 'file://{}'.format(input_path),
    
    23
    +                'filename': output_path,
    
    24
    +                'ref': utils.sha256sum(input_path),
    
    25
    +            }
    
    26
    +        ]
    
    27
    +    }
    
    28
    +
    
    29
    +
    
    30
    +@pytest.mark.datafiles(DATA_DIR)
    
    31
    +def test_source_checkout(datafiles, cli):
    
    32
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    33
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    34
    +    target = 'checkout-deps.bst'
    
    35
    +
    
    36
    +    result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
    
    37
    +    result.assert_success()
    
    38
    +
    
    39
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    40
    +
    
    41
    +
    
    42
    +@pytest.mark.datafiles(DATA_DIR)
    
    43
    +@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
    
    44
    +def test_source_checkout_deps(datafiles, cli, deps):
    
    45
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    46
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    47
    +    target = 'checkout-deps.bst'
    
    48
    +
    
    49
    +    result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
    
    50
    +    result.assert_success()
    
    51
    +
    
    52
    +    # Sources of the target
    
    53
    +    if deps == 'build':
    
    54
    +        assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
    
    55
    +    else:
    
    56
    +        assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    57
    +
    
    58
    +    # Sources of the target's build dependencies
    
    59
    +    if deps in ('build', 'all'):
    
    60
    +        assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
    
    61
    +    else:
    
    62
    +        assert not os.path.exists(os.path.join(checkout, 'import-dev'))
    
    63
    +
    
    64
    +    # Sources of the target's runtime dependencies
    
    65
    +    if deps in ('run', 'all'):
    
    66
    +        assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
    
    67
    +    else:
    
    68
    +        assert not os.path.exists(os.path.join(checkout, 'import-bin'))
    
    69
    +
    
    70
    +
    
    71
    +@pytest.mark.datafiles(DATA_DIR)
    
    72
    +def test_source_checkout_except(datafiles, cli):
    
    73
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    74
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    75
    +    target = 'checkout-deps.bst'
    
    76
    +
    
    77
    +    result = cli.run(project=project, args=['source-checkout', target,
    
    78
    +                                            '--deps', 'all',
    
    79
    +                                            '--except', 'import-bin.bst',
    
    80
    +                                            checkout])
    
    81
    +    result.assert_success()
    
    82
    +
    
    83
    +    # Sources for the target should be present
    
    84
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    85
    +
    
    86
    +    # Sources for import-bin.bst should not be present
    
    87
    +    assert not os.path.exists(os.path.join(checkout, 'import-bin'))
    
    88
    +
    
    89
    +    # Sources for other dependencies should be present
    
    90
    +    assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
    
    91
    +
    
    92
    +
    
    93
    +@pytest.mark.datafiles(DATA_DIR)
    
    94
    +@pytest.mark.parametrize('fetch', [(False), (True)])
    
    95
    +def test_source_checkout_fetch(datafiles, cli, fetch):
    
    96
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    97
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    98
    +    target = 'remote-import-dev.bst'
    
    99
    +    target_path = os.path.join(project, 'elements', target)
    
    100
    +
    
    101
    +    # Create an element with remote source
    
    102
    +    element = generate_remote_import_element(
    
    103
    +        os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
    
    104
    +        'pony.h')
    
    105
    +    _yaml.dump(element, target_path)
    
    106
    +
    
    107
    +    # Testing --fetch option requires that we do not have the sources
    
    108
    +    # cached already
    
    109
    +    assert cli.get_element_state(project, target) == 'fetch needed'
    
    110
    +
    
    111
    +    args = ['source-checkout']
    
    112
    +    if fetch:
    
    113
    +        args += ['--fetch']
    
    114
    +    args += [target, checkout]
    
    115
    +    result = cli.run(project=project, args=args)
    
    116
    +
    
    117
    +    if fetch:
    
    118
    +        result.assert_success()
    
    119
    +        assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))
    
    120
    +    else:
    
    121
    +        result.assert_main_error(ErrorDomain.PIPELINE, 'uncached-sources')



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]