[Notes] [Git][BuildStream/buildstream][issue-638-validate-all-files] 4 commits: _project.py: Validate nodes early in Project._load



Title: GitLab

Phillip Smyth pushed to branch issue-638-validate-all-files at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -166,6 +166,18 @@ original_main = click.BaseCommand.main
    166 166
     click.BaseCommand.main = override_main
    
    167 167
     
    
    168 168
     
    
    169
    +def get_files(directory):
    
    170
    +    output = tuple()
    
    171
    +    directory = os.path.abspath(directory)
    
    172
    +    for root, _, files in os.walk(directory):
    
    173
    +        for file in files:
    
    174
    +            if not file.startswith(".") and not file.endswith("~"):
    
    175
    +                relDir = os.path.relpath(root, directory)
    
    176
    +                relFile = os.path.join(relDir, file).strip("./")
    
    177
    +                output = output + (relFile,)
    
    178
    +    return output
    
    179
    +
    
    180
    +
    
    169 181
     ##################################################################
    
    170 182
     #                          Main Options                          #
    
    171 183
     ##################################################################
    
    ... ... @@ -487,6 +499,8 @@ def push(app, elements, deps, remote):
    487 499
     @click.option('--deps', '-d', default='all',
    
    488 500
                   type=click.Choice(['none', 'plan', 'run', 'build', 'all']),
    
    489 501
                   help='The dependencies to show (default: all)')
    
    502
    +@click.option('--all', 'all_', default=False, is_flag=True,
    
    503
    +              help="Validate all files in project")
    
    490 504
     @click.option('--order', default="stage",
    
    491 505
                   type=click.Choice(['stage', 'alpha']),
    
    492 506
                   help='Staging or alphabetic ordering of dependencies')
    
    ... ... @@ -496,7 +510,7 @@ def push(app, elements, deps, remote):
    496 510
     @click.argument('elements', nargs=-1,
    
    497 511
                     type=click.Path(readable=False))
    
    498 512
     @click.pass_obj
    
    499
    -def show(app, elements, deps, except_, order, format_):
    
    513
    +def show(app, elements, deps, except_, order, format_, all_):
    
    500 514
         """Show elements in the pipeline
    
    501 515
     
    
    502 516
         By default this will show all of the dependencies of the
    
    ... ... @@ -543,6 +557,13 @@ def show(app, elements, deps, except_, order, format_):
    543 557
             bst show target.bst --format \\
    
    544 558
                 $'---------- %{name} ----------\\n%{vars}'
    
    545 559
         """
    
    560
    +    if all_ and elements:
    
    561
    +        click.echo("ERROR: You can not use --all if you have defined an element", err=True)
    
    562
    +        sys.exit(-1)
    
    563
    +
    
    564
    +    if all_:
    
    565
    +        elements = get_files("elements")
    
    566
    +
    
    546 567
         with app.initialized():
    
    547 568
             dependencies = app.stream.load_selection(elements,
    
    548 569
                                                      selection=deps,
    

  • buildstream/_project.py
    ... ... @@ -219,6 +219,19 @@ class Project():
    219 219
     
    
    220 220
             return self._cache_key
    
    221 221
     
    
    222
    +    def _validate_node(self, node):
    
    223
    +        _yaml.node_validate(node, [
    
    224
    +            'format-version',
    
    225
    +            'element-path', 'variables',
    
    226
    +            'environment', 'environment-nocache',
    
    227
    +            'split-rules', 'elements', 'plugins',
    
    228
    +            'aliases', 'name',
    
    229
    +            'artifacts', 'options',
    
    230
    +            'fail-on-overlap', 'shell', 'fatal-warnings',
    
    231
    +            'ref-storage', 'sandbox', 'mirrors', 'remote-execution',
    
    232
    +            'sources', '(@)' 
    
    233
    +        ])
    
    234
    +
    
    222 235
         # create_element()
    
    223 236
         #
    
    224 237
         # Instantiate and return an element
    
    ... ... @@ -402,6 +415,8 @@ class Project():
    402 415
                     "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
    
    403 416
                     .format(format_version, major, minor, BST_FORMAT_VERSION))
    
    404 417
     
    
    418
    +        self._validate_node(pre_config_node)
    
    419
    +
    
    405 420
             # FIXME:
    
    406 421
             #
    
    407 422
             #   Performing this check manually in the absense
    
    ... ... @@ -467,16 +482,7 @@ class Project():
    467 482
     
    
    468 483
             self._load_pass(config, self.config)
    
    469 484
     
    
    470
    -        _yaml.node_validate(config, [
    
    471
    -            'format-version',
    
    472
    -            'element-path', 'variables',
    
    473
    -            'environment', 'environment-nocache',
    
    474
    -            'split-rules', 'elements', 'plugins',
    
    475
    -            'aliases', 'name',
    
    476
    -            'artifacts', 'options',
    
    477
    -            'fail-on-overlap', 'shell', 'fatal-warnings',
    
    478
    -            'ref-storage', 'sandbox', 'mirrors', 'remote-execution'
    
    479
    -        ])
    
    485
    +        self._validate_node(config)
    
    480 486
     
    
    481 487
             #
    
    482 488
             # Now all YAML composition is done, from here on we just load
    

  • tests/frontend/invalid_element_path/project.conf
    1
    +# Project config for frontend build test
    
    2
    +name: test
    
    3
    +
    
    4
    +elephant-path: elements

  • tests/frontend/project_fail/elements/compose-all.bst
    1
    +kind: compose
    
    2
    +
    
    3
    +depends:
    
    4
    +- fileNAME: import-dev.bst
    
    5
    +  type: build
    
    6
    +
    
    7
    +config:
    
    8
    +  # Dont try running the sandbox, we dont have a
    
    9
    +  # runtime to run anything in this context.
    
    10
    +  integrate: False

  • tests/frontend/project_fail/elements/import-dev.bst
    1
    +kind: import
    
    2
    +sources:
    
    3
    +- kind: local
    
    4
    +  path: files/dev-files

  • tests/frontend/project_fail/elements/target.bst
    1
    +kind: stack
    
    2
    +description: |
    
    3
    +
    
    4
    +  Main stack target for the bst build test
    
    5
    +
    
    6
    +depends:
    
    7
    +- compose-all.bst

  • tests/frontend/project_fail/files/dev-files/usr/include/pony.h
    1
    +#ifndef __PONY_H__
    
    2
    +#define __PONY_H__
    
    3
    +
    
    4
    +#define PONY_BEGIN "Once upon a time, there was a pony."
    
    5
    +#define PONY_END "And they lived happily ever after, the end."
    
    6
    +
    
    7
    +#define MAKE_PONY(story)  \
    
    8
    +  PONY_BEGIN \
    
    9
    +  story \
    
    10
    +  PONY_END
    
    11
    +
    
    12
    +#endif /* __PONY_H__ */

  • tests/frontend/project_fail/project.conf
    1
    +# Project config for frontend build test
    
    2
    +name: test
    
    3
    +
    
    4
    +element-path: elements

  • tests/frontend/show.py
    ... ... @@ -36,6 +36,30 @@ def test_show(cli, datafiles, target, format, expected):
    36 36
                                  .format(expected, result.output))
    
    37 37
     
    
    38 38
     
    
    39
    +@pytest.mark.datafiles(os.path.join(
    
    40
    +    os.path.dirname(os.path.realpath(__file__)),
    
    41
    +    "invalid_element_path",
    
    42
    +))
    
    43
    +def test_show_invalid_element_path(cli, datafiles):
    
    44
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    45
    +    result = cli.run(project=project, silent=True, args=[
    
    46
    +        'show',
    
    47
    +        "foo.bst"])
    
    48
    +
    
    49
    +
    
    50
    +@pytest.mark.datafiles(DATA_DIR + "_fail")
    
    51
    +def test_show_fail(cli, datafiles):
    
    52
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    53
    +    prev_dir = os.getcwd()
    
    54
    +    os.chdir(project)
    
    55
    +    result = cli.run(project=project, silent=True, args=[
    
    56
    +        'show',
    
    57
    +        '--all'])
    
    58
    +    os.chdir(prev_dir)
    
    59
    +
    
    60
    +    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
    
    61
    +
    
    62
    +
    
    39 63
     @pytest.mark.datafiles(DATA_DIR)
    
    40 64
     @pytest.mark.parametrize("target,except_,expected", [
    
    41 65
         ('target.bst', 'import-bin.bst', ['import-dev.bst', 'compose-all.bst', 'target.bst']),
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]