GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/81-precon-fail] 8 commits: README.rst: Remove badges (moved to header)

Title: GitLab

finn pushed to branch finn/81-precon-fail at BuildGrid / buildgrid

Commits:

17 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -33,6 +33,7 @@ before_script:
    33 33 
         - ${BGD} server start buildgrid/_app/settings/default.yml &
    34 34 
         - sleep 1 # Allow server to boot
    35 35 
         - ${BGD} bot dummy &
    36 
    +    - ${BGD} cas upload-dummy
    36 37 
         - ${BGD} execute request-dummy --wait-for-completion
    37 38
    38 39

  • README.rst
    1 1
    2 
    -.. image:: https://gitlab.com/Buildgrid/buildgrid/badges/master/pipeline.svg
    3 
    -   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    4 
    -
    5 
    -.. image:: https://gitlab.com/BuildGrid/buildgrid/badges/master/coverage.svg?job=coverage
    6 
    -   :target: https://buildgrid.gitlab.io/buildgrid/coverage
    7 
    -
    8 
    -
    9 2 
     .. _about:
    10 3
    11 
    -About BuildGrid
    12 
    -===============
    4 
    +About
    5 
    +=====
    13 6
    14 7
    15 8 
     .. _what-is-it:

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -65,6 +65,23 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    65 65 
         context.logger.debug("Starting for remote {}".format(context.remote))
    66 66
    67 67
    68 
    +@cli.command('upload-dummy', short_help="Upload a dummy action. Should be used with `execute dummy-request`")
    69 
    +@pass_context
    70 
    +def upload_dummy(context):
    71 
    +    context.logger.info("Uploading dummy action...")
    72 
    +    action = remote_execution_pb2.Action(do_not_cache=True)
    73 
    +    action_digest = create_digest(action.SerializeToString())
    74 
    +
    75 
    +    request = remote_execution_pb2.BatchUpdateBlobsRequest(instance_name=context.instance_name)
    76 
    +    request.requests.add(digest=action_digest,
    77 
    +                         data=action.SerializeToString())
    78 
    +
    79 
    +    stub = remote_execution_pb2_grpc.ContentAddressableStorageStub(context.channel)
    80 
    +    response = stub.BatchUpdateBlobs(request)
    81 
    +
    82 
    +    context.logger.info(response)
    83 
    +
    84 
    +
    68 85 
     @cli.command('upload-files', short_help="Upload files to the CAS server.")
    69 86 
     @click.argument('files', nargs=-1, type=click.File('rb'), required=True)
    70 87 
     @pass_context

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -76,9 +76,11 @@ def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    76 76 
                   help="Stream updates until jobs are completed.")
    77 77 
     @pass_context
    78 78 
     def request_dummy(context, number, wait_for_completion):
    79 
    -    action_digest = remote_execution_pb2.Digest()
    80 79
    81 80 
         context.logger.info("Sending execution request...")
    81 
    +    action = remote_execution_pb2.Action(do_not_cache=True)
    82 
    +    action_digest = create_digest(action.SerializeToString())
    83 
    +
    82 84 
         stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    83 85
    84 86 
         request = remote_execution_pb2.ExecuteRequest(instance_name=context.instance_name,
    ... ... @@ -90,25 +92,20 @@ def request_dummy(context, number, wait_for_completion):
    90 92 
             responses.append(stub.Execute(request))
    91 93
    92 94 
         for response in responses:
    95 
    +
    93 96 
             if wait_for_completion:
    97 
    +            result = None
    94 98 
                 for stream in response:
    95 
    -                context.logger.info(stream)
    96 
    -        else:
    97 
    -            context.logger.info(next(response))
    99 
    +                result = stream
    100 
    +                context.logger.info(result)
    98 101
    102 
    +            if not result.done:
    103 
    +                click.echo("Result did not return True." +
    104 
    +                           "Was the action uploaded to CAS?", err=True)
    105 
    +                sys.exit(-1)
    99 106
    100 
    -@cli.command('wait', short_help="Streams an operation until it is complete.")
    101 
    -@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    102 
    -@pass_context
    103 
    -def wait_execution(context, operation_name):
    104 
    -    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    105 
    -    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    106 
    -                                                        name=operation_name)
    107 
    -
    108 
    -    response = stub.WaitExecution(request)
    109 
    -
    110 
    -    for stream in response:
    111 
    -        context.logger.info(stream)
    107 
    +        else:
    108 
    +            context.logger.info(next(response))
    112 109
    113 110
    114 111 
     @cli.command('command', short_help="Send a command to be executed.")

  • buildgrid/_app/commands/cmd_operation.py
    ... ... @@ -27,12 +27,13 @@ import sys
    27 27 
     import click
    28 28 
     import grpc
    29 29
    30 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    30 31 
     from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    31 32
    32 33 
     from ..cli import pass_context
    33 34
    34 35
    35 
    -@click.group(name='operation', short_help="Long running operations commands")
    36 
    +@click.group(name='operation', short_help="Long running operations commands.")
    36 37 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    37 38 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    38 39 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    ... ... @@ -93,3 +94,17 @@ def lists(context):
    93 94
    94 95 
         for op in response.operations:
    95 96 
             context.logger.info(op)
    97 
    +
    98 
    +
    99 
    +@cli.command('wait', short_help="Streams an operation until it is complete.")
    100 
    +@click.argument('operation-name', nargs=1, type=click.STRING, required=True)
    101 
    +@pass_context
    102 
    +def wait(context, operation_name):
    103 
    +    stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    104 
    +    request = remote_execution_pb2.WaitExecutionRequest(instance_name=context.instance_name,
    105 
    +                                                        name=operation_name)
    106 
    +
    107 
    +    response = stub.WaitExecution(request)
    108 
    +
    109 
    +    for stream in response:
    110 
    +        context.logger.info(stream)

  • buildgrid/server/_exceptions.py
    ... ... @@ -46,3 +46,12 @@ class OutOfRangeError(BgdError):
    46 46
    47 47 
         def __init__(self, message, detail=None, reason=None):
    48 48 
             super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
    49 
    +
    50 
    +
    51 
    +class FailedPreconditionError(BgdError):
    52 
    +    """ One or more errors occurred in setting up the action requested, such as a missing input
    53 
    +    or command or no worker being available. The client may be able to fix the errors and retry.
    54 
    +    """
    55 
    +
    56 
    +    def __init__(self, message, detail=None, reason=None):
    57 
    +        super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)

  • buildgrid/server/execution/instance.py
    ... ... @@ -24,12 +24,12 @@ import logging
    24 24 
     from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Action
    25 25
    26 26 
     from ..job import Job
    27 
    -from .._exceptions import InvalidArgumentError
    27 
    +from .._exceptions import InvalidArgumentError, FailedPreconditionError
    28 28
    29 29
    30 30 
     class ExecutionInstance:
    31 31
    32 
    -    def __init__(self, scheduler, storage=None):
    32 
    +    def __init__(self, scheduler, storage):
    33 33 
             self.logger = logging.getLogger(__name__)
    34 34 
             self._storage = storage
    35 35 
             self._scheduler = scheduler
    ... ... @@ -43,13 +43,12 @@ class ExecutionInstance:
    43 43 
             this action.
    44 44 
             """
    45 45
    46 
    -        do_not_cache = False
    47 
    -        if self._storage is not None:
    48 
    -            action = self._storage.get_message(action_digest, Action)
    49 
    -            if action is not None:
    50 
    -                do_not_cache = action.do_not_cache
    46 
    +        action = self._storage.get_message(action_digest, Action)
    51 47
    52 
    -        job = Job(action_digest, do_not_cache, message_queue)
    48 
    +        if not action:
    49 
    +            raise FailedPreconditionError("Could not get action from storage.")
    50 
    +
    51 
    +        job = Job(action_digest, action.do_not_cache, message_queue)
    53 52 
             self.logger.info("Operation name: [{}]".format(job.name))
    54 53
    55 54 
             self._scheduler.append_job(job, skip_cache_lookup)

  • buildgrid/server/execution/service.py
    ... ... @@ -30,7 +30,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
    30 30
    31 31 
     from buildgrid._protos.google.longrunning import operations_pb2
    32 32
    33 
    -from .._exceptions import InvalidArgumentError
    33 
    +from .._exceptions import InvalidArgumentError, FailedPreconditionError
    34 34
    35 35
    36 36 
     class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    ... ... @@ -63,6 +63,12 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    63 63 
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    64 64 
                 yield operations_pb2.Operation()
    65 65
    66 
    +        except FailedPreconditionError as e:
    67 
    +            self.logger.error(e)
    68 
    +            context.set_details(str(e))
    69 
    +            context.set_code(grpc.StatusCode.FAILED_PRECONDITION)
    70 
    +            yield operations_pb2.Operation()
    71 
    +
    66 72 
         def WaitExecution(self, request, context):
    67 73 
             try:
    68 74 
                 names = request.name.split("/")

  • docs/source/data/bazel-example-server.conf
    1 1 
     server:
    2 
    -  port: 50051
    3 
    -  insecure-mode: true
    2 
    +  - !channel
    3 
    +    port: 50051
    4 
    +    insecure_mode: true
    4 5
    5 6 
     instances:
    6 7 
       - name: main

  • docs/source/data/buildstream-example-server.conf
    1 
    +server:
    2 
    +  - !channel
    3 
    +    port: 50051
    4 
    +    insecure_mode: true
    5 
    +
    6 
    +instances:
    7 
    +  - name: ""
    8 
    +
    9 
    +    storages:
    10 
    +      - !lru-storage &main-storage
    11 
    +        size: 512MB
    12 
    +
    13 
    +    services:
    14 
    +      - !action-cache &main-action
    15 
    +        storage: *main-storage
    16 
    +        max_cached_refs: 256
    17 
    +        allow_updates: true
    18 
    +      - !execution
    19 
    +        storage: *main-storage
    20 
    +        action_cache: *main-action
    21 
    +      - !cas
    22 
    +        storage: *main-storage
    23 
    +      - !reference-cache
    24 
    +        storage: *main-storage
    25 
    +        max_cached_refs: 128
    26 
    +      - !bytestream
    27 
    +        storage: *main-storage
    \ No newline at end of file

  • docs/source/reference_cli.rst
    1 1
    2 
    -.. _cli_reference:
    2 
    +.. _cli-reference:
    3 3
    4 4 
     CLI reference
    5 5 
     =============
    ... ... @@ -8,112 +8,118 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    8 8
    9 9 
     ----
    10 10
    11 
    -.. _invoking_bgd:
    11 
    +.. _invoking-bgd:
    12 12
    13 13 
     .. click:: buildgrid._app:cli
    14 14 
        :prog: bgd
    15 15
    16 16 
     ----
    17 17
    18 
    -.. _invoking_bgd_bot:
    18 
    +.. _invoking-bgd-bot:
    19 19
    20 20 
     .. click:: buildgrid._app.commands.cmd_bot:cli
    21 21 
        :prog: bgd bot
    22 22
    23 23 
     ----
    24 24
    25 
    -.. _invoking_bgd_bot_buildbox:
    25 
    +.. _invoking-bgd-bot-buildbox:
    26 26
    27 27 
     .. click:: buildgrid._app.commands.cmd_bot:run_buildbox
    28 28 
        :prog: bgd bot buildbot
    29 29
    30 30 
     ----
    31 31
    32 
    -.. _invoking_bgd_bot_dummy:
    32 
    +.. _invoking-bgd-bot-dummy:
    33 33
    34 34 
     .. click:: buildgrid._app.commands.cmd_bot:run_dummy
    35 35 
        :prog: bgd bot dummy
    36 36
    37 37 
     ----
    38 38
    39 
    -.. _invoking_bgd_bot_temp_directory:
    39 
    +.. _invoking-bgd-bot-temp-directory:
    40 40
    41 41 
     .. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
    42 42 
        :prog: bgd bot temp-directory
    43 43
    44 44 
     ----
    45 45
    46 
    -.. _invoking_bgd_cas:
    46 
    +.. _invoking-bgd-cas:
    47 47
    48 48 
     .. click:: buildgrid._app.commands.cmd_cas:cli
    49 49 
        :prog: bgd cas
    50 50
    51 51 
     ----
    52 52
    53 
    -.. _invoking_bgd_cas_upload_dir:
    53 
    +.. _invoking-bgd-cas-upload-dir:
    54 54
    55 55 
     .. click:: buildgrid._app.commands.cmd_cas:upload_dir
    56 56 
        :prog: bgd cas upload-dir
    57 57
    58 58 
     ----
    59 59
    60 
    -.. _invoking_bgd_cas_upload_files:
    60 
    +.. _invoking-bgd-cas-upload-files:
    61 61
    62 62 
     .. click:: buildgrid._app.commands.cmd_cas:upload_files
    63 63 
        :prog: bgd cas upload-files
    64 64
    65 65 
     ----
    66 66
    67 
    -.. _invoking_bgd_execute:
    67 
    +.. _invoking-bgd-execute:
    68 68
    69 69 
     .. click:: buildgrid._app.commands.cmd_execute:cli
    70 70 
        :prog: bgd execute
    71 71
    72 72 
     ----
    73 73
    74 
    -.. _invoking_bgd_execute_command:
    74 
    +.. _invoking-bgd-execute-command:
    75 75
    76 76 
     .. click:: buildgrid._app.commands.cmd_execute:command
    77 77 
        :prog: bgd execute command
    78 78
    79 79 
     ----
    80 80
    81 
    -.. _invoking_bgd_execute_list:
    81 
    +.. _invoking-bgd-execute-request-dummy:
    82 82
    83 
    -.. click:: buildgrid._app.commands.cmd_execute:list_operations
    84 
    -   :prog: bgd execute list
    83 
    +.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    84 
    +   :prog: bgd execute request-dummy
    85 85
    86 86 
     ----
    87 87
    88 
    -.. _invoking_bgd_execute_ request_dummy:
    88 
    +.. _invoking-bgd-operation:
    89 89
    90 
    -.. click:: buildgrid._app.commands.cmd_execute:request_dummy
    91 
    -   :prog: bgd execute request-dummy
    90 
    +.. click:: buildgrid._app.commands.cmd_operation:cli
    91 
    +   :prog: bgd operation
    92 92
    93 93 
     ----
    94 94
    95 
    -.. _invoking_bgd_execute_status:
    95 
    +.. _invoking-bgd-operation-list:
    96 96
    97 
    -.. click:: buildgrid._app.commands.cmd_execute:operation_status
    98 
    -   :prog: bgd execute status
    97 
    +.. click:: buildgrid._app.commands.cmd_operation:lists
    98 
    +   :prog: bgd operation list
    99 99
    100 100 
     ----
    101 101
    102 
    -.. _invoking_bgd_execute_wait:
    102 
    +.. _invoking-bgd-operation-status:
    103 103
    104 
    -.. click:: buildgrid._app.commands.cmd_execute:wait_execution
    105 
    -   :prog: bgd execute wait
    104 
    +.. click:: buildgrid._app.commands.cmd_operation:status
    105 
    +   :prog: bgd operation status
    106 106
    107 107 
     ----
    108 108
    109 
    -.. _invoking_bgd_server:
    109 
    +.. _invoking-bgd-operation-wait:
    110 
    +
    111 
    +.. click:: buildgrid._app.commands.cmd_operation:wait
    112 
    +   :prog: bgd operation wait
    113 
    +
    114 
    +----
    115 
    +.. _invoking-bgd-server:
    110 116
    111 117 
     .. click:: buildgrid._app.commands.cmd_server:cli
    112 118 
        :prog: bgd server
    113 119
    114 120 
     ----
    115 121
    116 
    -.. _invoking_bgd_server_start:
    122 
    +.. _invoking-bgd-server-start:
    117 123
    118 124 
     .. click:: buildgrid._app.commands.cmd_server:start
    119 125 
        :prog: bgd server start

  • docs/source/using.rst
    ... ... @@ -7,7 +7,8 @@ Using
    7 7 
     This section covers how to run an use the BuildGrid build service.
    8 8
    9 9 
     .. toctree::
    10 
    -   :maxdepth: 2
    10 
    +   :maxdepth: 3
    11 11
    12 12 
        using_internal.rst
    13 13 
        using_bazel.rst
    14 
    +   using_buildstream.rst

  • docs/source/using_bazel.rst
    1 1
    2 
    -.. _bazel-builds:
    2 
    +.. _bazel-client:
    3 3
    4 
    -Bazel builds
    4 
    +Bazel client
    5 5 
     ============
    6 6
    7 7 
     `Bazel`_ is a *“fast, scalable, multi-language and extensible build system”*
    ... ... @@ -18,7 +18,7 @@ Configuration
    18 18 
     -------------
    19 19
    20 20 
     Bazel accepts many options that can either be specified as command line
    21 
    -arguments when involking the ``bazel`` tool or stored in a `.bazelrc`_
    21 
    +arguments when invoking the ``bazel`` tool or stored in a `.bazelrc`_
    22 22 
     configuration file. In order to activate remote build execution, the
    23 23 
     ``bazel build`` subcommand needs to be given specific `build options`_,
    24 24 
     including:
    ... ... @@ -65,7 +65,7 @@ Next, change the current directory to the Bazel workspace root:
    65 65
    66 66 
        cd bazel-examples/cpp-tutorial/stage3
    67 67
    68 
    -.. note::
    68 
    +.. hint::
    69 69
    70 70 
        All the commands in the instructions below are expected to be executed from
    71 71 
        that root directory (the stage3 example workspace's root directory).
    ... ... @@ -80,7 +80,7 @@ below, paste it in a ``server.conf`` file in the root directory:
    80 80 
     This defines a single ``main`` server instance implementing a
    81 81 
     ``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
    82 82 
     ``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
    83 
    -You can then start the BuildGrid server deamon using that configuration by
    83 
    +You can then start the BuildGrid server daemon using that configuration by
    84 84 
     running:
    85 85
    86 86 
     .. code-block:: sh
    ... ... @@ -97,8 +97,10 @@ has ``gcc`` installed, run:
    97 97 
        bgd bot --remote=http://localhost:50051 --parent=main temp-directory
    98 98
    99 99 
     The ``--remote`` option is used to specify the server location (running on the
    100 
    -same machine here, and listenning to port 50051). The ``--parent`` option is
    101 
    -used to specifiy the server instance you except the bot to be attached to.
    100 
    +same machine here, and listening to port 50051). The ``--parent`` option is used
    101 
    +to specify the server instance you except the bot to be attached to. Refer to
    102 
    +the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
    103 
    +line interface details.
    102 104
    103 105 
     The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
    104 106 
     some :ref:`configuration <bazel-configuration>` in order to run remote builds.
    ... ... @@ -109,7 +111,7 @@ file in the root directory:
    109 111
    110 112 
        build --spawn_strategy=remote --genrule_strategy=remote --remote_executor=localhost:50051 --remote_instance_name=main
    111 113
    112 
    -This ativates Bazel's remote execution mode and points to the ``main`` remote
    114 
    +This activates Bazel's remote execution mode and points to the ``main`` remote
    113 115 
     execution server instance at ``localhost:50051``.
    114 116
    115 117 
     You can finally have Bazel to build the example workspace by running:
    ... ... @@ -118,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
    118 120
    119 121 
        bazel build //main:hello-world
    120 122
    121 
    -You can verify that the example has been successfully build by runnig the
    123 
    +You can verify that the example has been successfully build by running the
    122 124 
     generated executable. Simply invoke:
    123 125
    124 126 
     .. code-block:: sh

  • docs/source/using_buildstream.rst
    1 
    +
    2 
    +.. _buildstream-client:
    3 
    +
    4 
    +BuildStream client
    5 
    +==================
    6 
    +
    7 
    +`BuildStream`_  is a free software tool for building and integrating software
    8 
    +stacks. It supports remote build execution using the remote execution API
    9 
    +(REAPI) v2.
    10 
    +
    11 
    +.. note::
    12 
    +
    13 
    +   There is no stable release of BuildStream with support for remote execution
    14 
    +   yet. You'll have to `install it from sources`_ in order to build with
    15 
    +   remote execution.
    16 
    +
    17 
    +.. _BuildStream: https://buildstream.build
    18 
    +.. _install it from sources: https://buildstream.build/source_install.html
    19 
    +
    20 
    +
    21 
    +.. _buildstream-configuration:
    22 
    +
    23 
    +Configuration
    24 
    +-------------
    25 
    +
    26 
    +BuildStream uses `YAML`_ for build definition and configuration. It has two
    27 
    +levels of configuration: user and project level. `User-level configuration`_ is
    28 
    +stored in your ``buildstream.conf`` file while `project-level configuration`_
    29 
    +is defined in each project's ``project.conf`` file.
    30 
    +
    31 
    +.. note::
    32 
    +
    33 
    +   At the moment, remote execution can only be configured at project-level.
    34 
    +
    35 
    +.. _YAML: http://yaml.org
    36 
    +.. _User-level configuration: https://buildstream.gitlab.io/buildstream/using_config.html
    37 
    +.. _project-level configuration: https://buildstream.gitlab.io/buildstream/format_project.html
    38 
    +
    39 
    +
    40 
    +Project configuration
    41 
    +~~~~~~~~~~~~~~~~~~~~~
    42 
    +
    43 
    +In order to activate remote build execution at project-level, the project's
    44 
    +``project.conf`` file must declare two specific configuration nodes:
    45 
    +
    46 
    +- ``artifacts`` for `remote CAS endpoint details`_.
    47 
    +- ``remote-execution`` for `remote execution endpoint details`_.
    48 
    +
    49 
    +.. important::
    50 
    +
    51 
    +   BuildStream does not support multi-instance remote execution servers and will
    52 
    +   always submit remote execution request omitting the instance name parameter.
    53 
    +   Thus, you must declare an unnamed `""` instance  in your server configuration
    54 
    +   to workaround this.
    55 
    +
    56 
    +.. important::
    57 
    +
    58 
    +   If you are using BuildGrid's artifact server, the server instance **must**
    59 
    +   accept pushes from your client for remote execution to be possible.
    60 
    +
    61 
    +
    62 
    +.. _remote CAS endpoint details: https://buildstream.gitlab.io/buildstream/install_artifacts.html#user-configuration
    63 
    +.. _remote execution endpoint details: https://buildstream.gitlab.io/buildstream/format_project.html#remote-execution
    64 
    +
    65 
    +
    66 
    +BuildBox tool
    67 
    +~~~~~~~~~~~~~
    68 
    +
    69 
    +BuildStream performs builds in a `sandbox`_ on top of a project-defined
    70 
    +environment, not relying on any host-tools. BuildGrid supports this kind of
    71 
    +build using the ``buildbox`` bot, a specific type of bot relying on `BuildBox`_
    72 
    +for build execution.
    73 
    +
    74 
    +BuildBox can execute build commands in a sandbox on top of an environment
    75 
    +constructed from provided sources using `FUSE`_. It also uses `bubblewrap`_
    76 
    +for sandboxing without requiring root privileges.
    77 
    +
    78 
    +BuildBox being a rather young project, it isn't packaged yet and you'll have to
    79 
    +build it from source. You may want follow the `manual instructions`_ or you can
    80 
    +build it with BuildStream using the `dedicated integration project`_ (recommanded).
    81 
    +
    82 
    +.. important::
    83 
    +
    84 
    +   Whatever the method you use to install BuildBox, you also have to install
    85 
    +   bubblewrap along, minimum required version being `0.1.8`_.
    86 
    +
    87 
    +.. _sandbox: https://buildstream.gitlab.io/buildstream/additional_sandboxing.html
    88 
    +.. _BuildBox: https://gitlab.com/BuildStream/buildbox
    89 
    +.. _FUSE: https://en.wikipedia.org/wiki/Filesystem_in_Userspace
    90 
    +.. _bubblewrap: https://github.com/projectatomic/bubblewrap
    91 
    +.. _manual instructions: https://gitlab.com/BuildStream/buildbox/blob/master/INSTALL.rst
    92 
    +.. _dedicated integration project: https://gitlab.com/BuildStream/buildbox-integration
    93 
    +.. _0.1.8: https://github.com/projectatomic/bubblewrap/releases/tag/v0.1.8
    94 
    +
    95 
    +
    96 
    +.. _buildstream-example:
    97 
    +
    98 
    +Example build
    99 
    +-------------
    100 
    +
    101 
    +The BuildStream repository contains `example projects`_ used for testing purpose
    102 
    +in the project's `usage documentation section`_. We'll focus here on
    103 
    +instructions on how to build the `autotools example`_ running BuildStream and
    104 
    +BuildGrid on your local machine, compiling the `GNU Automake`_ hello example
    105 
    +program in a sandbox on top of a minimal `Alpine Linux`_ environment.
    106 
    +
    107 
    +First, you need to checkout the buildstream repository sources:
    108 
    +
    109 
    +.. code-block:: sh
    110 
    +
    111 
    +   git clone https://gitlab.com/BuildStream/buildstream.git
    112 
    +
    113 
    +Next, change the current directory to the BuildStream project root:
    114 
    +
    115 
    +.. code-block:: sh
    116 
    +
    117 
    +   cd buildstream/doc/examples/autotools
    118 
    +
    119 
    +.. hint::
    120 
    +
    121 
    +   All the commands in the instructions below are expected to be executed from
    122 
    +   that root directory (the autotools example project's root directory).
    123 
    +
    124 
    +Before running BuildStream and building the example project, you'll have to setup
    125 
    +and run a BuildGrid server and bot. A minimal server's configuration is given
    126 
    +below, paste it in a ``server.conf`` file in the root directory:
    127 
    +
    128 
    +.. literalinclude:: ./data/buildstream-example-server.conf
    129 
    +   :language: yaml
    130 
    +
    131 
    +This defines a single unnamed server instance implementing a
    132 
    +``ContentAddressableStorage`` (CAS) + ``Reference`` + ``ByteStream`` service
    133 
    +together with an ``Execution`` + ``ActionCache`` service, both using the
    134 
    +same in-memory storage. You can then start the BuildGrid server daemon using
    135 
    +that configuration by running:
    136 
    +
    137 
    +.. code-block:: sh
    138 
    +
    139 
    +   bgd server start server.conf
    140 
    +
    141 
    +In order to perform the actual build work, you need to attach a ``buildbox``
    142 
    +worker bot to that server for that unnamed instance. Once you've make sure
    143 
    +that the ``buildbox`` tool is functional on your machine (refer to
    144 
    +:ref:`configuration <buildstream-configuration>`), run:
    145 
    +
    146 
    +.. code-block:: sh
    147 
    +
    148 
    +   bgd bot --remote=http://localhost:50051 --parent= buildbox --fuse-dir=fuse --local-cas=cache
    149 
    +
    150 
    +The ``--remote`` option is used to specify the server location (running on the
    151 
    +same machine here, and listening to port 50051). The ``--parent`` option is
    152 
    +used to specify the server instance you except the bot to be attached to (empty
    153 
    +here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
    154 
    +and respectively specify the sandbox mount point and local CAS cache locations.
    155 
    +Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
    156 
    +for command line interface details.
    157 
    +
    158 
    +The BuildGrid server is now ready to accept jobs and execute them. The example
    159 
    +project needs some :ref:`configuration <buildstream-configuration>` tweaks in
    160 
    +order to be build remotely. Below is the configuration fragment you should
    161 
    +append at the end of the ``project.conf`` file from the root directory:
    162 
    +
    163 
    +.. code-block:: yaml
    164 
    +
    165 
    +   artifacts:
    166 
    +     url: http://localhost:50051
    167 
    +     push: true
    168 
    +
    169 
    +   remote-execution:
    170 
    +     url: http://localhost:50051
    171 
    +
    172 
    +This activates BuildGrid's remote execution mode and points to the unnamed
    173 
    +remote execution server instance at ``localhost:50051``.
    174 
    +
    175 
    +You can finally have BuildStream to build the example project by running:
    176 
    +
    177 
    +.. code-block:: sh
    178 
    +
    179 
    +   bst build hello.bst
    180 
    +
    181 
    +You can verify that the example has been successfully build by running the
    182 
    +generated executable. Simply invoke:
    183 
    +
    184 
    +.. code-block:: sh
    185 
    +
    186 
    +   bst shell hello.bst -- hello
    187 
    +
    188 
    +.. _example projects: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples
    189 
    +.. _usage documentation section: http://buildstream.gitlab.io/buildstream/main_using.html
    190 
    +.. _autotools example: https://gitlab.com/BuildStream/buildstream/tree/master/doc/examples/autotools
    191 
    +.. _GNU Automake: https://www.gnu.org/software/automake
    192 
    +.. _Alpine Linux: https://www.alpinelinux.org

  • docs/source/using_internal.rst
    1 
    +.. _internal-client:
    1 2
    2 
    -.. _internal-builds:
    3 3
    4 
    -Internal builds
    4 
    +Internal client
    5 5 
     ===============
    6 6
    7 
    +BuildGrid contains a minimal remote execution client that can be used through
    8 
    +the ``bgd`` command line interface.
    9 
    +
    7 10
    8 11 
     .. _dummy-test:
    9 12
    ... ... @@ -16,7 +19,13 @@ In one terminal, start a server:
    16 19
    17 20 
        bgd server start buildgrid/_app/settings/default.yml
    18 21
    19 
    -In another terminal, send a request for work:
    22 
    +In another terminal, upload an action to CAS:
    23 
    +
    24 
    +.. code-block::sh
    25 
    +
    26 
    +   bgd cas upload-dummy
    27 
    +
    28 
    +Then send a request for work:
    20 29
    21 30 
     .. code-block:: sh
    22 31

  • tests/integration/execution_service.py
    ... ... @@ -28,6 +28,7 @@ import pytest
    28 28 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    29 29 
     from buildgrid._protos.google.longrunning import operations_pb2
    30 30
    31 
    +from buildgrid.utils import create_digest
    31 32 
     from buildgrid.server import job
    32 33 
     from buildgrid.server.controller import ExecutionController
    33 34 
     from buildgrid.server.cas.storage import lru_memory_cache
    ... ... @@ -37,6 +38,8 @@ from buildgrid.server.execution.service import ExecutionService
    37 38
    38 39
    39 40 
     server = mock.create_autospec(grpc.server)
    41 
    +action = remote_execution_pb2.Action(do_not_cache=True)
    42 
    +action_digest = create_digest(action.SerializeToString())
    40 43
    41 44
    42 45 
     @pytest.fixture
    ... ... @@ -47,12 +50,16 @@ def context():
    47 50
    48 51 
     @pytest.fixture(params=["action-cache", "no-action-cache"])
    49 52 
     def controller(request):
    53 
    +    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
    54 
    +    write_session = storage.begin_write(action_digest)
    55 
    +    storage.commit_write(action_digest, write_session)
    56 
    +
    50 57 
         if request.param == "action-cache":
    51 
    -        storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
    52 58 
             cache = ActionCache(storage, 50)
    53 59 
             yield ExecutionController(cache, storage)
    60 
    +
    54 61 
         else:
    55 
    -        yield ExecutionController()
    62 
    +        yield ExecutionController(None, storage)
    56 63
    57 64
    58 65 
     # Instance to test
    ... ... @@ -66,9 +73,6 @@ def instance(controller):
    66 73
    67 74 
     @pytest.mark.parametrize("skip_cache_lookup", [True, False])
    68 75 
     def test_execute(skip_cache_lookup, instance, context):
    69 
    -    action_digest = remote_execution_pb2.Digest()
    70 
    -    action_digest.hash = 'zhora'
    71 
    -
    72 76 
         request = remote_execution_pb2.ExecuteRequest(instance_name='',
    73 77 
                                                       action_digest=action_digest,
    74 78 
                                                       skip_cache_lookup=skip_cache_lookup)
    ... ... @@ -91,10 +95,16 @@ def test_wrong_execute_instance(instance, context):
    91 95 
         context.set_code.assert_called_once_with(grpc.StatusCode.INVALID_ARGUMENT)
    92 96
    93 97
    94 
    -def test_wait_execution(instance, controller, context):
    95 
    -    action_digest = remote_execution_pb2.Digest()
    96 
    -    action_digest.hash = 'zhora'
    98 
    +def test_no_action_digest_in_storage(instance, context):
    99 
    +    request = remote_execution_pb2.ExecuteRequest(instance_name='',
    100 
    +                                                  skip_cache_lookup=True)
    101 
    +    response = instance.Execute(request, context)
    102 
    +
    103 
    +    next(response)
    104 
    +    context.set_code.assert_called_once_with(grpc.StatusCode.FAILED_PRECONDITION)
    97 105
    106 
    +
    107 
    +def test_wait_execution(instance, controller, context):
    98 108 
         j = job.Job(action_digest, None)
    99 109 
         j._operation.done = True
    100 110

  • tests/integration/operations_service.py
    ... ... @@ -24,18 +24,21 @@ import grpc
    24 24 
     from grpc._server import _Context
    25 25 
     import pytest
    26 26
    27 
    -from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    28 
    -from buildgrid._protos.google.longrunning import operations_pb2
    29 
    -
    27 
    +from buildgrid.utils import create_digest
    30 28 
     from buildgrid.server.controller import ExecutionController
    31 
    -from buildgrid.server._exceptions import InvalidArgumentError
    32 
    -
    29 
    +from buildgrid.server.cas.storage import lru_memory_cache
    33 30 
     from buildgrid.server.operations import service
    34 31 
     from buildgrid.server.operations.service import OperationsService
    32 
    +from buildgrid.server._exceptions import InvalidArgumentError
    33 
    +
    34 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    35 
    +from buildgrid._protos.google.longrunning import operations_pb2
    35 36
    36 37
    37 38 
     server = mock.create_autospec(grpc.server)
    38 39 
     instance_name = "blade"
    40 
    +action = remote_execution_pb2.Action(do_not_cache=True)
    41 
    +action_digest = create_digest(action.SerializeToString())
    39 42
    40 43
    41 44 
     # Can mock this
    ... ... @@ -47,9 +50,6 @@ def context():
    47 50 
     # Requests to make
    48 51 
     @pytest.fixture
    49 52 
     def execute_request():
    50 
    -    action_digest = remote_execution_pb2.Digest()
    51 
    -    action_digest.hash = 'zhora'
    52 
    -
    53 53 
         yield remote_execution_pb2.ExecuteRequest(instance_name='',
    54 54 
                                                   action_digest=action_digest,
    55 55 
                                                   skip_cache_lookup=True)
    ... ... @@ -57,7 +57,11 @@ def execute_request():
    57 57
    58 58 
     @pytest.fixture
    59 59 
     def controller():
    60 
    -    yield ExecutionController()
    60 
    +    storage = lru_memory_cache.LRUMemoryCache(1024 * 1024)
    61 
    +    write_session = storage.begin_write(action_digest)
    62 
    +    storage.commit_write(action_digest, write_session)
    63 
    +
    64 
    +    yield ExecutionController(None, storage)
    61 65
    62 66
    63 67 
     # Instance to test

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]