[Notes] [Git][BuildStream/buildstream][chandan/source-checkout] 4 commits: element.py: Fix cache check in non-strict mode



Title: GitLab

Chandan Singh pushed to branch chandan/source-checkout at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -662,6 +662,30 @@ def checkout(app, element, location, force, deps, integrate, hardlinks, tar):
    662 662
                                 tar=tar)
    
    663 663
     
    
    664 664
     
    
    665
    +##################################################################
    
    666
    +#                  Source Checkout Command                      #
    
    667
    +##################################################################
    
    668
    +@cli.command(name="source-checkout", short_help="Checkout sources for an element")
    
    669
    +@click.option('--except', 'except_', multiple=True,
    
    670
    +              type=click.Path(readable=False),
    
    671
    +              help="Except certain dependencies")
    
    672
    +@click.option('--deps', '-d', default='none',
    
    673
    +              type=click.Choice(['build', 'none', 'run', 'all']),
    
    674
    +              help='The dependencies whose sources to checkout (default: none)')
    
    675
    +@click.argument('element',
    
    676
    +                type=click.Path(readable=False))
    
    677
    +@click.argument('location', type=click.Path())
    
    678
    +@click.pass_obj
    
    679
    +def source_checkout(app, element, location, deps, except_):
    
    680
    +    """Checkout sources of an element to the specified location
    
    681
    +    """
    
    682
    +    with app.initialized():
    
    683
    +        app.stream.source_checkout(element,
    
    684
    +                                   location=location,
    
    685
    +                                   deps=deps,
    
    686
    +                                   except_targets=except_)
    
    687
    +
    
    688
    +
    
    665 689
     ##################################################################
    
    666 690
     #                      Workspace Command                         #
    
    667 691
     ##################################################################
    

  • buildstream/_stream.py
    ... ... @@ -381,27 +381,7 @@ class Stream():
    381 381
             elements, _ = self._load((target,), (), fetch_subprojects=True)
    
    382 382
             target = elements[0]
    
    383 383
     
    
    384
    -        if not tar:
    
    385
    -            try:
    
    386
    -                os.makedirs(location, exist_ok=True)
    
    387
    -            except OSError as e:
    
    388
    -                raise StreamError("Failed to create checkout directory: '{}'"
    
    389
    -                                  .format(e)) from e
    
    390
    -
    
    391
    -        if not tar:
    
    392
    -            if not os.access(location, os.W_OK):
    
    393
    -                raise StreamError("Checkout directory '{}' not writable"
    
    394
    -                                  .format(location))
    
    395
    -            if not force and os.listdir(location):
    
    396
    -                raise StreamError("Checkout directory '{}' not empty"
    
    397
    -                                  .format(location))
    
    398
    -        elif os.path.exists(location) and location != '-':
    
    399
    -            if not os.access(location, os.W_OK):
    
    400
    -                raise StreamError("Output file '{}' not writable"
    
    401
    -                                  .format(location))
    
    402
    -            if not force and os.path.exists(location):
    
    403
    -                raise StreamError("Output file '{}' already exists"
    
    404
    -                                  .format(location))
    
    384
    +        self.__check_location_writable(location, force=force, tar=tar)
    
    405 385
     
    
    406 386
             # Stage deps into a temporary sandbox first
    
    407 387
             try:
    
    ... ... @@ -438,6 +418,35 @@ class Stream():
    438 418
                 raise StreamError("Error while staging dependencies into a sandbox"
    
    439 419
                                   ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    440 420
     
    
    421
    +    # source_checkout()
    
    422
    +    #
    
    423
    +    # Checkout sources of the target element to the specified location
    
    424
    +    #
    
    425
    +    # Args:
    
    426
    +    #    target (str): The target element whose sources to checkout
    
    427
    +    #    location (str): Location to checkout the sources to
    
    428
    +    #    deps (str): The dependencies to checkout
    
    429
    +    #    except_targets (list): List of targets to except from staging
    
    430
    +    #
    
    431
    +    def source_checkout(self, target, *,
    
    432
    +                        location=None,
    
    433
    +                        deps='none',
    
    434
    +                        except_targets=()):
    
    435
    +
    
    436
    +        self.__check_location_writable(location)
    
    437
    +
    
    438
    +        elements, _ = self._load((target,), (),
    
    439
    +                                 selection=deps,
    
    440
    +                                 except_targets=except_targets,
    
    441
    +                                 fetch_subprojects=True)
    
    442
    +
    
    443
    +        # Stage all sources determined by scope
    
    444
    +        try:
    
    445
    +            self._write_element_sources(location, elements)
    
    446
    +        except BstError as e:
    
    447
    +            raise StreamError("Error while writing sources"
    
    448
    +                              ": '{}'".format(e), detail=e.detail, reason=e.reason) from e
    
    449
    +
    
    441 450
         # workspace_open
    
    442 451
         #
    
    443 452
         # Open a project workspace
    
    ... ... @@ -721,7 +730,7 @@ class Stream():
    721 730
                     if self._write_element_script(source_directory, element)
    
    722 731
                 ]
    
    723 732
     
    
    724
    -            self._write_element_sources(tempdir, elements)
    
    733
    +            self._write_element_sources(os.path.join(tempdir, "source"), elements)
    
    725 734
                 self._write_build_script(tempdir, elements)
    
    726 735
                 self._collect_sources(tempdir, tar_location,
    
    727 736
                                       target.normal_name, compression)
    
    ... ... @@ -1084,11 +1093,10 @@ class Stream():
    1084 1093
         # Write all source elements to the given directory
    
    1085 1094
         def _write_element_sources(self, directory, elements):
    
    1086 1095
             for element in elements:
    
    1087
    -            source_dir = os.path.join(directory, "source")
    
    1088
    -            element_source_dir = os.path.join(source_dir, element.normal_name)
    
    1089
    -            os.makedirs(element_source_dir)
    
    1090
    -
    
    1091
    -            element._stage_sources_at(element_source_dir)
    
    1096
    +            element_source_dir = os.path.join(directory, element.normal_name)
    
    1097
    +            if list(element.sources()):
    
    1098
    +                os.makedirs(element_source_dir)
    
    1099
    +                element._stage_sources_at(element_source_dir)
    
    1092 1100
     
    
    1093 1101
         # Write a master build script to the sandbox
    
    1094 1102
         def _write_build_script(self, directory, elements):
    
    ... ... @@ -1117,3 +1125,29 @@ class Stream():
    1117 1125
     
    
    1118 1126
                 with tarfile.open(tar_name, permissions) as tar:
    
    1119 1127
                     tar.add(directory, arcname=element_name)
    
    1128
    +
    
    1129
    +    #############################################################
    
    1130
    +    #                    Private Methods                        #
    
    1131
    +    #############################################################
    
    1132
    +
    
    1133
    +    # Check if given location is writable
    
    1134
    +    def __check_location_writable(self, location, force=False, tar=False):
    
    1135
    +        if not tar:
    
    1136
    +            try:
    
    1137
    +                os.makedirs(location, exist_ok=True)
    
    1138
    +            except OSError as e:
    
    1139
    +                raise StreamError("Failed to create checkout directory: '{}'"
    
    1140
    +                                  .format(e)) from e
    
    1141
    +            if not os.access(location, os.W_OK):
    
    1142
    +                raise StreamError("Checkout directory '{}' not writable"
    
    1143
    +                                  .format(location))
    
    1144
    +            if not force and os.listdir(location):
    
    1145
    +                raise StreamError("Checkout directory '{}' not empty"
    
    1146
    +                                  .format(location))
    
    1147
    +        elif os.path.exists(location) and location != '-':
    
    1148
    +            if not os.access(location, os.W_OK):
    
    1149
    +                raise StreamError("Output file '{}' not writable"
    
    1150
    +                                  .format(location))
    
    1151
    +            if not force and os.path.exists(location):
    
    1152
    +                raise StreamError("Output file '{}' already exists"
    
    1153
    +                                  .format(location))

  • buildstream/element.py
    ... ... @@ -1366,6 +1366,14 @@ class Element(Plugin):
    1366 1366
         def _stage_sources_at(self, vdirectory, mount_workspaces=True):
    
    1367 1367
             with self.timed_activity("Staging sources", silent_nested=True):
    
    1368 1368
     
    
    1369
    +            if self._get_consistency() != Consistency.CACHED:
    
    1370
    +                detail = "Sources for this element have not been catched yet.\n" + \
    
    1371
    +                         "Try fetching the sources first with `bst fetch`. \n" + \
    
    1372
    +                         "Alternatively, run this command with `--fetch` option."
    
    1373
    +                raise ElementError("Sources are not cached",
    
    1374
    +                                   detail=detail,
    
    1375
    +                                   reason="uncached-source-checkout-attempt")
    
    1376
    +
    
    1369 1377
                 if not isinstance(vdirectory, Directory):
    
    1370 1378
                     vdirectory = FileBasedDirectory(vdirectory)
    
    1371 1379
                 if not vdirectory.is_empty():
    
    ... ... @@ -2084,7 +2092,7 @@ class Element(Plugin):
    2084 2092
         #
    
    2085 2093
         # Raises an error if the artifact is not cached.
    
    2086 2094
         #
    
    2087
    -    def __assert_cached(self, keystrength=_KeyStrength.STRONG):
    
    2095
    +    def __assert_cached(self, keystrength=None):
    
    2088 2096
             assert self.__is_cached(keystrength=keystrength), "{}: Missing artifact {}".format(
    
    2089 2097
                 self, self._get_brief_display_key())
    
    2090 2098
     
    

  • tests/completions/completions.py
    ... ... @@ -15,6 +15,7 @@ MAIN_COMMANDS = [
    15 15
         'push ',
    
    16 16
         'shell ',
    
    17 17
         'show ',
    
    18
    +    'source-checkout ',
    
    18 19
         'source-bundle ',
    
    19 20
         'track ',
    
    20 21
         'workspace '
    

  • tests/frontend/project/elements/checkout-deps.bst
    1
    +kind: import
    
    2
    +description: It is important for this element to have both build and runtime dependencies
    
    3
    +sources:
    
    4
    +- kind: local
    
    5
    +  path: files/etc-files
    
    6
    +depends:
    
    7
    +- filename: import-dev.bst
    
    8
    +  type: build
    
    9
    +- filename: import-bin.bst
    
    10
    +  type: runtime

  • tests/frontend/project/elements/rebuild-target.bst
    1
    +kind: compose
    
    2
    +
    
    3
    +build-depends:
    
    4
    +- target.bst

  • tests/frontend/project/files/etc-files/etc/buildstream/config
    1
    +config

  • tests/frontend/rebuild.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +from tests.testutils import cli
    
    4
    +
    
    5
    +# Project directory
    
    6
    +DATA_DIR = os.path.join(
    
    7
    +    os.path.dirname(os.path.realpath(__file__)),
    
    8
    +    "project",
    
    9
    +)
    
    10
    +
    
    11
    +
    
    12
    +def strict_args(args, strict):
    
    13
    +    if strict != "strict":
    
    14
    +        return ['--no-strict'] + args
    
    15
    +    return args
    
    16
    +
    
    17
    +
    
    18
    +@pytest.mark.datafiles(DATA_DIR)
    
    19
    +@pytest.mark.parametrize("strict", ["strict", "non-strict"])
    
    20
    +def test_rebuild(datafiles, cli, strict):
    
    21
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    22
    +    checkout = os.path.join(cli.directory, 'checkout')
    
    23
    +
    
    24
    +    # First build intermediate target.bst
    
    25
    +    result = cli.run(project=project, args=strict_args(['build', 'target.bst'], strict))
    
    26
    +    result.assert_success()
    
    27
    +
    
    28
    +    # Modify base import
    
    29
    +    with open(os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'new.h'), "w") as f:
    
    30
    +        f.write("#define NEW")
    
    31
    +
    
    32
    +    # Rebuild base import and build top-level rebuild-target.bst
    
    33
    +    # In non-strict mode, this does not rebuild intermediate target.bst,
    
    34
    +    # which means that a weakly cached target.bst will be staged as dependency.
    
    35
    +    result = cli.run(project=project, args=strict_args(['build', 'rebuild-target.bst'], strict))
    
    36
    +    result.assert_success()

  • tests/frontend/source_checkout.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from tests.testutils import cli
    
    5
    +
    
    6
    +# Project directory
    
    7
    +DATA_DIR = os.path.join(
    
    8
    +    os.path.dirname(os.path.realpath(__file__)),
    
    9
    +    "project",
    
    10
    +)
    
    11
    +
    
    12
    +
    
    13
    +@pytest.mark.datafiles(DATA_DIR)
    
    14
    +def test_source_checkout(datafiles, cli):
    
    15
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    16
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    17
    +    target = 'checkout-deps.bst'
    
    18
    +
    
    19
    +    result = cli.run(project=project, args=['source-checkout', target, '--deps', 'none', checkout])
    
    20
    +    result.assert_success()
    
    21
    +
    
    22
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    23
    +
    
    24
    +
    
    25
    +@pytest.mark.datafiles(DATA_DIR)
    
    26
    +@pytest.mark.parametrize('deps', [("build"), ("none"), ("run"), ("all")])
    
    27
    +def test_source_checkout_deps(datafiles, cli, deps):
    
    28
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    29
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    30
    +    target = 'checkout-deps.bst'
    
    31
    +
    
    32
    +    result = cli.run(project=project, args=['source-checkout', target, '--deps', deps, checkout])
    
    33
    +    result.assert_success()
    
    34
    +
    
    35
    +    # Sources of the target
    
    36
    +    if deps == 'build':
    
    37
    +        assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
    
    38
    +    else:
    
    39
    +        assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    40
    +
    
    41
    +    # Sources of the target's build dependencies
    
    42
    +    if deps in ('build', 'all'):
    
    43
    +        assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
    
    44
    +    else:
    
    45
    +        assert not os.path.exists(os.path.join(checkout, 'import-dev'))
    
    46
    +
    
    47
    +    # Sources of the target's runtime dependencies
    
    48
    +    if deps in ('run', 'all'):
    
    49
    +        assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
    
    50
    +    else:
    
    51
    +        assert not os.path.exists(os.path.join(checkout, 'import-bin'))
    
    52
    +
    
    53
    +
    
    54
    +@pytest.mark.datafiles(DATA_DIR)
    
    55
    +def test_source_checkout_except(datafiles, cli):
    
    56
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    57
    +    checkout = os.path.join(cli.directory, 'source-checkout')
    
    58
    +    target = 'checkout-deps.bst'
    
    59
    +
    
    60
    +    result = cli.run(project=project, args=['source-checkout', target,
    
    61
    +                                            '--deps', 'all',
    
    62
    +                                            '--except', 'import-bin.bst',
    
    63
    +                                            checkout])
    
    64
    +    result.assert_success()
    
    65
    +
    
    66
    +    # Sources for the target should be present
    
    67
    +    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))
    
    68
    +
    
    69
    +    # Sources for import-bin.bst should not be present
    
    70
    +    assert not os.path.exists(os.path.join(checkout, 'import-bin'))
    
    71
    +
    
    72
    +    # Sources for other dependencies should be present
    
    73
    +    assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]