GNOME.org
 

[Notes] [Git][BuildStream/buildstream][Qinusty/235-manifest] 2 commits: cli.py: Add --build-manifest to build command

Title: GitLab

Qinusty pushed to branch Qinusty/235-manifest at BuildStream / buildstream

Commits:

6 changed files:

Changes:

  • buildstream/_frontend/cli.py
    ... ... @@ -3,6 +3,7 @@ import sys
    3 3
    4 4 
     import click
    5 5 
     from .. import _yaml
    6 
    +from .. import _manifest
    6 7 
     from .._exceptions import BstError, LoadError, AppError
    7 8 
     from .._versions import BST_FORMAT_VERSION
    8 9 
     from .complete import main_bashcomplete, complete_path, CompleteUnhandled
    ... ... @@ -289,6 +290,15 @@ def init(app, project_name, format_version, element_path, force):
    289 290 
     ##################################################################
    290 291 
     #                          Build Command                         #
    291 292 
     ##################################################################
    293 
    +def _validate_manifest_path(ctx, param, value):
    294 
    +    if not value:
    295 
    +        return
    296 
    +    if value.lower().endswith(".yaml") or value.lower().endswith(".yml"):
    297 
    +        return os.path.abspath(value)
    298 
    +    else:
    299 
    +        raise click.BadParameter("Manifest files are outputted as YAML\n\t" +
    300 
    +                                 "Please provide a path with a valid file extension (yml, yaml)")
    301 
    +
    292 302 
     @cli.command(short_help="Build elements in a pipeline")
    293 303 
     @click.option('--all', 'all_', default=False, is_flag=True,
    294 304 
                   help="Build elements that would not be needed for the current build plan")
    ... ... @@ -305,10 +315,16 @@ def init(app, project_name, format_version, element_path, force):
    305 315 
                   help="Allow tracking to cross junction boundaries")
    306 316 
     @click.option('--track-save', default=False, is_flag=True,
    307 317 
                   help="Deprecated: This is ignored")
    318 
    +@click.option('--build-manifest', default=False, is_flag=True,
    319 
    +              help="Produces a build manifest containing elements and sources.")
    320 
    +@click.option('--manifest-path', default=None, type=click.Path(readable=False),
    321 
    +              help="Provides a path for a build manifest to be written to.",
    322 
    +              callback=_validate_manifest_path)
    308 323 
     @click.argument('elements', nargs=-1,
    309 324 
                     type=click.Path(readable=False))
    310 325 
     @click.pass_obj
    311 
    -def build(app, elements, all_, track_, track_save, track_all, track_except, track_cross_junctions):
    326 
    +def build(app, elements, all_, track_, track_save, build_manifest,
    327 
    +          manifest_path, track_all, track_except, track_cross_junctions):
    312 328 
         """Build elements in a pipeline"""
    313 329
    314 330 
         if (track_except or track_cross_junctions) and not (track_ or track_all):
    ... ... @@ -329,6 +345,13 @@ def build(app, elements, all_, track_, track_save, track_all, track_except, trac
    329 345 
                              track_cross_junctions=track_cross_junctions,
    330 346 
                              build_all=all_)
    331 347
    348 
    +        if build_manifest and not manifest_path:
    349 
    +            manifest_path = os.path.join(app.project.directory,
    350 
    +                                         "build_manifest.yaml")
    351 
    +        if manifest_path:
    352 
    +            _manifest.generate(app.context, app.stream.total_elements,
    353 
    +                               app._session_start, manifest_path)
    354 
    +
    332 355
    333 356 
     ##################################################################
    334 357 
     #                          Fetch Command                         #

  • buildstream/_manifest.py
    1 
    +#
    2 
    +#  Copyright (C) 2018 Codethink Limited
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Josh Smith <josh smith codethink co uk>
    19 
    +
    20 
    +from ruamel import yaml
    21 
    +from . import __version__ as bst_version
    22 
    +from . import _yaml
    23 
    +from ._message import MessageType, Message
    24 
    +
    25 
    +# generate():
    26 
    +#
    27 
    +# Generates a manfiest file from the collection of elements.
    28 
    +#
    29 
    +# Args:
    30 
    +#    context (Context): Application context
    31 
    +#    elements (list of Element): Collection of elements to build the manifest from
    32 
    +#    build_start_datetime (datetime): The start of the build assosciated with this manifest
    33 
    +#    manifest_path (str): Absolute path to write the manifest file to.
    34 
    +#
    35 
    +def generate(context, elements, build_start_datetime, manifest_path):
    36 
    +    with context.timed_activity("Building Manifest"):
    37 
    +        manifest = _build(elements, build_start_datetime)
    38 
    +
    39 
    +    _yaml.dump(manifest, manifest_path)
    40 
    +    context.message(Message(None, MessageType.STATUS,
    41 
    +                            "Manifest saved to {}".format(manifest_path)))
    42 
    +
    43 
    +# _build():
    44 
    +#
    45 
    +# Builds a manifest (dictionary) using the provided elements to be stored.
    46 
    +#
    47 
    +# Args:
    48 
    +#    elements (list of Element): Collection of elements to build the manifest from
    49 
    +#    build_start_datetime (datetime): The start of the build assosciated with this manifest
    50 
    +#
    51 
    +# Returns:
    52 
    +#    (CommentedMap): A dictionary containing the entire
    53 
    +#                    manifest produced from the provided elements.
    54 
    +#
    55 
    +def _build(elements, build_start_datetime):
    56 
    +    manifest = yaml.comments.CommentedMap()
    57 
    +
    58 
    +    # Add BuildStream Version
    59 
    +    manifest['BuildStream_Version'] = "{}".format(bst_version)
    60 
    +    # Add Build Date
    61 
    +    manifest['Build_Date'] = build_start_datetime.isoformat()
    62 
    +    manifest['Elements'] = yaml.comments.CommentedMap()
    63 
    +
    64 
    +    # Sort elements
    65 
    +    elements = sorted(elements, key=lambda e: e.normal_name)
    66 
    +
    67 
    +    # Add Elements
    68 
    +    for elem in elements:
    69 
    +        manifest['Elements'][elem.normal_name] = _build_element(elem)
    70 
    +
    71 
    +    return manifest
    72 
    +
    73 
    +# _build_element():
    74 
    +#
    75 
    +# Builds a manifest segment for an individual element.
    76 
    +#
    77 
    +# Args:
    78 
    +#    element (Element): Element to extract information from
    79 
    +#
    80 
    +# Returns:
    81 
    +#    (CommentedMap): A dictionary containing the information
    82 
    +#                    extracted from the provided element
    83 
    +#
    84 
    +def _build_element(element):
    85 
    +    element_dict = yaml.comments.CommentedMap()
    86 
    +    sources = yaml.comments.CommentedMap()
    87 
    +    # Add Cache Key
    88 
    +    cache_key = element._get_cache_key()
    89 
    +    if cache_key:
    90 
    +        element_dict["Cache_Key"] = cache_key
    91 
    +
    92 
    +    # Add sources
    93 
    +    for source in element.sources():
    94 
    +        src = _build_source(source)
    95 
    +        if src:
    96 
    +            source_desc = "{}({})".format(source._get_element_index(), type(source).__name__)
    97 
    +            sources[source_desc] = src
    98 
    +    if sources:
    99 
    +        element_dict['Sources'] = sources
    100 
    +
    101 
    +
    102 
    +    return element_dict
    103 
    +
    104 
    +# _build_source():
    105 
    +#
    106 
    +# Builds a manifest segment for an individual source.
    107 
    +#
    108 
    +# Args:
    109 
    +#    source (Source): Source to extract information from
    110 
    +#
    111 
    +# Returns:
    112 
    +#    (CommentedMap): A dictionary containing the information
    113 
    +#                    extracted from the provided source
    114 
    +#
    115 
    +def _build_source(source):
    116 
    +    src = yaml.comments.CommentedMap()
    117 
    +    if hasattr(source, "url") and source.url:
    118 
    +        src["url"] = source.url
    119 
    +    if hasattr(source, "ref") and source.ref:
    120 
    +        src["ref"] = source.ref
    121 
    +    if hasattr(source, "path") and source.path:
    122 
    +        src["path"] = source.path
    123 
    +
    124 
    +    return src if src else None

  • buildstream/source.py
    ... ... @@ -786,6 +786,9 @@ class Source(Plugin):
    786 786 
             else:
    787 787 
                 return None
    788 788
    789 
    +    def _get_element_index(self):
    790 
    +        return self.__element_index
    791 
    +
    789 792 
         #############################################################
    790 793 
         #                   Local Private Methods                   #
    791 794 
         #############################################################

  • tests/manifest/manifest.py
    1 
    +import pytest
    2 
    +import os
    3 
    +from ruamel import yaml
    4 
    +
    5 
    +from tests.testutils import cli
    6 
    +
    7 
    +# Project directory
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project",
    11 
    +)
    12 
    +
    13 
    +
    14 
    +@pytest.mark.datafiles(DATA_DIR)
    15 
    +@pytest.mark.parametrize("specify_path, build_manifest", [
    16 
    +    (True, True), (True, False), (False, True)
    17 
    +])
    18 
    +def test_manifest_created(tmpdir, cli, datafiles, specify_path, build_manifest):
    19 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    20 
    +
    21 
    +    manifest_path = os.path.join(str(tmpdir), "build_manifest.yaml")
    22 
    +
    23 
    +    args = ['build', "base.bst"]
    24 
    +
    25 
    +    if specify_path:
    26 
    +        args += ["--manifest-path", manifest_path]
    27 
    +    if build_manifest:
    28 
    +        args.append("--build-manifest")
    29 
    +
    30 
    +    result = cli.run(project=project, args=args)
    31 
    +    result.assert_success()
    32 
    +
    33 
    +    with open(manifest_path) as f:
    34 
    +        manifest = yaml.load(f, Loader=yaml.loader.RoundTripLoader)
    35 
    +
    36 
    +    assert len(manifest["Elements"]["base"]["Sources"]) == 1
    37 
    +
    38 
    +
    39 
    +@pytest.mark.datafiles(DATA_DIR)
    40 
    +@pytest.mark.parametrize("extension, valid", [
    41 
    +    (".yaml", True),
    42 
    +    (".yml", True),
    43 
    +    (".bst", False),
    44 
    +    (".ynl", False),
    45 
    +    (".xml", False),
    46 
    +    (".mnf", False),
    47 
    +    (".txt", False),
    48 
    +    (".abc", False),
    49 
    +    (".json", False)
    50 
    +])
    51 
    +def test_manifest_extensions(tmpdir, cli, datafiles, extension, valid):
    52 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    53 
    +
    54 
    +    manifest_path = os.path.join(str(tmpdir), "build_manifest{}" + extension)
    55 
    +
    56 
    +    result = cli.run(project=project, args=['build', "base.bst", "--manifest-path", manifest_path])
    57 
    +
    58 
    +    if valid:
    59 
    +        result.assert_success()
    60 
    +    else:
    61 
    +        assert result.exit_code == 2

  • tests/manifest/project/elements/base.bst
    1 
    +kind: import
    2 
    +description: Custom foo element
    3 
    +
    4 
    +sources:
    5 
    +  - kind: local
    6 
    +    path: files/hello
    \ No newline at end of file

  • tests/manifest/project/project.conf
    1 
    +# Project config for frontend build test
    2 
    +name: test
    3 
    +
    4 
    +element-path: elements

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]