[Notes] [Git][BuildGrid/buildgrid][mablanch/connection-drop-simulation] 11 commits: Locked sphinx to version 1.7.8.



Title: GitLab

Martin Blanchard pushed to branch mablanch/connection-drop-simulation at BuildGrid / buildgrid

Commits:

21 changed files:

Changes:

  • buildgrid/_app/bots/temp_directory.pybuildgrid/_app/bots/host.py
    ... ... @@ -24,7 +24,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p
    24 24
     from buildgrid.utils import output_file_maker, output_directory_maker
    
    25 25
     
    
    26 26
     
    
    27
    -def work_temp_directory(context, lease):
    
    27
    +def work_host_tools(context, lease):
    
    28 28
         """Executes a lease for a build action, using host tools.
    
    29 29
         """
    
    30 30
     
    

  • buildgrid/_app/commands/cmd_bot.py
    ... ... @@ -31,7 +31,7 @@ import grpc
    31 31
     from buildgrid.bot import bot, bot_interface
    
    32 32
     from buildgrid.bot.bot_session import BotSession, Device, Worker
    
    33 33
     
    
    34
    -from ..bots import buildbox, dummy, temp_directory
    
    34
    +from ..bots import buildbox, dummy, host
    
    35 35
     from ..cli import pass_context
    
    36 36
     
    
    37 37
     
    
    ... ... @@ -135,8 +135,7 @@ def cli(context, parent, remote, client_key, client_cert, server_cert,
    135 135
     @pass_context
    
    136 136
     def run_dummy(context):
    
    137 137
         """
    
    138
    -    Simple dummy client. Creates a session, accepts leases, does fake work and
    
    139
    -    updates the server.
    
    138
    +    Creates a session, accepts leases, does fake work and updates the server.
    
    140 139
         """
    
    141 140
         try:
    
    142 141
             b = bot.Bot(context.bot_session)
    
    ... ... @@ -146,15 +145,16 @@ def run_dummy(context):
    146 145
             pass
    
    147 146
     
    
    148 147
     
    
    149
    -@cli.command('temp-directory', short_help="Runs commands in temp directory and uploads results.")
    
    148
    +@cli.command('host-tools', short_help="Runs commands using the host's tools.")
    
    150 149
     @pass_context
    
    151
    -def run_temp_directory(context):
    
    152
    -    """ Downloads files and command from CAS and runs
    
    153
    -    in a temp directory, uploading result back to CAS
    
    150
    +def run_host_tools(context):
    
    151
    +    """
    
    152
    +    Downloads inputs from CAS, runs build commands using host-tools and uploads
    
    153
    +    result back to CAS.
    
    154 154
         """
    
    155 155
         try:
    
    156 156
             b = bot.Bot(context.bot_session)
    
    157
    -        b.session(temp_directory.work_temp_directory,
    
    157
    +        b.session(host.work_host_tools,
    
    158 158
                       context)
    
    159 159
         except KeyboardInterrupt:
    
    160 160
             pass
    
    ... ... @@ -168,7 +168,7 @@ def run_temp_directory(context):
    168 168
     @pass_context
    
    169 169
     def run_buildbox(context, local_cas, fuse_dir):
    
    170 170
         """
    
    171
    -    Uses BuildBox to run commands.
    
    171
    +    Uses BuildBox to run build commands.
    
    172 172
         """
    
    173 173
         context.local_cas = local_cas
    
    174 174
         context.fuse_dir = fuse_dir
    

  • buildgrid/_app/settings/parser.py
    ... ... @@ -35,6 +35,9 @@ from ..cli import Context
    35 35
     
    
    36 36
     
    
    37 37
     class YamlFactory(yaml.YAMLObject):
    
    38
    +    """ Base class for contructing maps or scalars from tags.
    
    39
    +    """
    
    40
    +
    
    38 41
         @classmethod
    
    39 42
         def from_yaml(cls, loader, node):
    
    40 43
             if isinstance(node, yaml.ScalarNode):
    
    ... ... @@ -47,6 +50,21 @@ class YamlFactory(yaml.YAMLObject):
    47 50
     
    
    48 51
     
    
    49 52
     class Channel(YamlFactory):
    
    53
    +    """Creates a GRPC channel.
    
    54
    +
    
    55
    +    The :class:`Channel` class returns a `grpc.Channel` and is generated from the tag ``!channel``.
    
    56
    +    Creates either a secure or insecure channel.
    
    57
    +
    
    58
    +    Args:
    
    59
    +       port (int): A port for the channel.
    
    60
    +       insecure_mode (bool): If ``True``, generates an insecure channel, even if there are
    
    61
    +    credentials. Defaults to ``True``.
    
    62
    +       credentials (dict, optional): A dictionary in the form::
    
    63
    +
    
    64
    +           tls-server-key: /path/to/server-key
    
    65
    +           tls-server-cert: /path/to/server-cert
    
    66
    +           tls-client-certs: /path/to/client-certs
    
    67
    +    """
    
    50 68
     
    
    51 69
         yaml_tag = u'!channel'
    
    52 70
     
    
    ... ... @@ -69,6 +87,13 @@ class Channel(YamlFactory):
    69 87
     
    
    70 88
     
    
    71 89
     class ExpandPath(YamlFactory):
    
    90
    +    """Returns a string of the user's path after expansion.
    
    91
    +
    
    92
    +    The :class:`ExpandPath` class returns a string and is generated from the tag ``!expand-path``.
    
    93
    +
    
    94
    +    Args:
    
    95
    +       path (str): Can be used with strings such as: ``~/dir/to/something`` or ``$HOME/certs``
    
    96
    +    """
    
    72 97
     
    
    73 98
         yaml_tag = u'!expand-path'
    
    74 99
     
    
    ... ... @@ -79,14 +104,30 @@ class ExpandPath(YamlFactory):
    79 104
     
    
    80 105
     
    
    81 106
     class Disk(YamlFactory):
    
    107
    +    """Generates :class:`buildgrid.server.cas.storage.disk.DiskStorage` using the tag ``!disk-storage``.
    
    108
    +
    
    109
    +    Args:
    
    110
    +       path (str): Path to directory to storage.
    
    111
    +
    
    112
    +    """
    
    82 113
     
    
    83 114
         yaml_tag = u'!disk-storage'
    
    84 115
     
    
    85 116
         def __new__(cls, path):
    
    117
    +        """Creates a new disk
    
    118
    +
    
    119
    +        Args:
    
    120
    +           path (str): Some path
    
    121
    +        """
    
    86 122
             return DiskStorage(path)
    
    87 123
     
    
    88 124
     
    
    89 125
     class LRU(YamlFactory):
    
    126
    +    """Generates :class:`buildgrid.server.cas.storage.lru_memory_cache.LRUMemoryCache` using the tag ``!lru-storage``.
    
    127
    +
    
    128
    +    Args:
    
    129
    +      size (int): Size e.g ``10kb``. Size parsed with :meth:`buildgrid._app.settings.parser._parse_size`.
    
    130
    +    """
    
    90 131
     
    
    91 132
         yaml_tag = u'!lru-storage'
    
    92 133
     
    
    ... ... @@ -95,6 +136,12 @@ class LRU(YamlFactory):
    95 136
     
    
    96 137
     
    
    97 138
     class S3(YamlFactory):
    
    139
    +    """Generates :class:`buildgrid.server.cas.storage.s3.S3Storage` using the tag ``!s3-storage``.
    
    140
    +
    
    141
    +    Args:
    
    142
    +        bucket (str): Name of bucket
    
    143
    +        endpoint (str): URL of endpoint.
    
    144
    +    """
    
    98 145
     
    
    99 146
         yaml_tag = u'!s3-storage'
    
    100 147
     
    
    ... ... @@ -103,6 +150,18 @@ class S3(YamlFactory):
    103 150
     
    
    104 151
     
    
    105 152
     class Remote(YamlFactory):
    
    153
    +    """Generates :class:`buildgrid.server.cas.storage.remote.RemoteStorage`
    
    154
    +    using the tag ``!remote-storage``.
    
    155
    +
    
    156
    +    Args:
    
    157
    +      url (str): URL to remote storage. If used with ``https``, needs credentials.
    
    158
    +      instance_name (str): Instance of the remote to connect to.
    
    159
    +      credentials (dict, optional): A dictionary in the form::
    
    160
    +
    
    161
    +           tls-client-key: /path/to/client-key
    
    162
    +           tls-client-cert: /path/to/client-cert
    
    163
    +           tls-server-cert: /path/to/server-cert
    
    164
    +    """
    
    106 165
     
    
    107 166
         yaml_tag = u'!remote-storage'
    
    108 167
     
    
    ... ... @@ -144,6 +203,18 @@ class Remote(YamlFactory):
    144 203
     
    
    145 204
     
    
    146 205
     class WithCache(YamlFactory):
    
    206
    +    """Generates :class:`buildgrid.server.cas.storage.with_cache.WithCacheStorage`
    
    207
    +    using the tag ``!with-cache-storage``.
    
    208
    +
    
    209
    +    Args:
    
    210
    +      url (str): URL to remote storage. If used with ``https``, needs credentials.
    
    211
    +      instance_name (str): Instance of the remote to connect to.
    
    212
    +      credentials (dict, optional): A dictionary in the form::
    
    213
    +
    
    214
    +           tls-client-key: /path/to/certs
    
    215
    +           tls-client-cert: /path/to/certs
    
    216
    +           tls-server-cert: /path/to/certs
    
    217
    +    """
    
    147 218
     
    
    148 219
         yaml_tag = u'!with-cache-storage'
    
    149 220
     
    
    ... ... @@ -152,6 +223,13 @@ class WithCache(YamlFactory):
    152 223
     
    
    153 224
     
    
    154 225
     class Execution(YamlFactory):
    
    226
    +    """Generates :class:`buildgrid.server.execution.service.ExecutionService`
    
    227
    +    using the tag ``!execution``.
    
    228
    +
    
    229
    +    Args:
    
    230
    +      storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
    
    231
    +      action_cache(:class:`Action`): Instance of action cache to use.
    
    232
    +    """
    
    155 233
     
    
    156 234
         yaml_tag = u'!execution'
    
    157 235
     
    
    ... ... @@ -160,6 +238,14 @@ class Execution(YamlFactory):
    160 238
     
    
    161 239
     
    
    162 240
     class Action(YamlFactory):
    
    241
    +    """Generates :class:`buildgrid.server.actioncache.service.ActionCacheService`
    
    242
    +    using the tag ``!action-cache``.
    
    243
    +
    
    244
    +    Args:
    
    245
    +      storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
    
    246
    +      max_cached_refs(int): Max number of cached actions.
    
    247
    +      allow_updates(bool): Allow updates pushed to CAS. Defaults to ``True``.
    
    248
    +    """
    
    163 249
     
    
    164 250
         yaml_tag = u'!action-cache'
    
    165 251
     
    
    ... ... @@ -168,6 +254,14 @@ class Action(YamlFactory):
    168 254
     
    
    169 255
     
    
    170 256
     class Reference(YamlFactory):
    
    257
    +    """Generates :class:`buildgrid.server.referencestorage.service.ReferenceStorageService`
    
    258
    +    using the tag ``!reference-cache``.
    
    259
    +
    
    260
    +    Args:
    
    261
    +      storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
    
    262
    +      max_cached_refs(int): Max number of cached actions.
    
    263
    +      allow_updates(bool): Allow updates pushed to CAS. Defauled to ``True``.
    
    264
    +    """
    
    171 265
     
    
    172 266
         yaml_tag = u'!reference-cache'
    
    173 267
     
    
    ... ... @@ -176,6 +270,12 @@ class Reference(YamlFactory):
    176 270
     
    
    177 271
     
    
    178 272
     class CAS(YamlFactory):
    
    273
    +    """Generates :class:`buildgrid.server.cas.service.ContentAddressableStorageService`
    
    274
    +    using the tag ``!cas``.
    
    275
    +
    
    276
    +    Args:
    
    277
    +      storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
    
    278
    +    """
    
    179 279
     
    
    180 280
         yaml_tag = u'!cas'
    
    181 281
     
    
    ... ... @@ -184,6 +284,12 @@ class CAS(YamlFactory):
    184 284
     
    
    185 285
     
    
    186 286
     class ByteStream(YamlFactory):
    
    287
    +    """Generates :class:`buildgrid.server.cas.service.ByteStreamService`
    
    288
    +    using the tag ``!bytestream``.
    
    289
    +
    
    290
    +    Args:
    
    291
    +      storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
    
    292
    +    """
    
    187 293
     
    
    188 294
         yaml_tag = u'!bytestream'
    
    189 295
     
    

  • buildgrid/client/cas.py
    ... ... @@ -283,7 +283,7 @@ class Downloader:
    283 283
                 try:
    
    284 284
                     batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
    
    285 285
                     for response in batch_response.responses:
    
    286
    -                    assert response.digest.hash in digests
    
    286
    +                    assert response.digest in digests
    
    287 287
     
    
    288 288
                         read_blobs.append(response.data)
    
    289 289
     
    

  • buildgrid/server/actioncache/service.py
    ... ... @@ -52,7 +52,7 @@ class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer):
    52 52
                 context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
    
    53 53
     
    
    54 54
             except NotFoundError as e:
    
    55
    -            self.logger.error(e)
    
    55
    +            self.logger.info(e)
    
    56 56
                 context.set_code(grpc.StatusCode.NOT_FOUND)
    
    57 57
     
    
    58 58
             return remote_execution_pb2.ActionResult()
    

  • buildgrid/server/cas/storage/disk.py
    ... ... @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk.
    21 21
     """
    
    22 22
     
    
    23 23
     import os
    
    24
    -import pathlib
    
    25 24
     import tempfile
    
    26 25
     
    
    27 26
     from .storage_abc import StorageABC
    
    ... ... @@ -30,28 +29,41 @@ from .storage_abc import StorageABC
    30 29
     class DiskStorage(StorageABC):
    
    31 30
     
    
    32 31
         def __init__(self, path):
    
    33
    -        self._path = pathlib.Path(path)
    
    34
    -        os.makedirs(str(self._path / "temp"), exist_ok=True)
    
    32
    +        if not os.path.isabs(path):
    
    33
    +            self.__root_path = os.path.abspath(path)
    
    34
    +        else:
    
    35
    +            self.__root_path = path
    
    36
    +        self.__cas_path = os.path.join(self.__root_path, 'cas')
    
    37
    +
    
    38
    +        self.objects_path = os.path.join(self.__cas_path, 'objects')
    
    39
    +        self.temp_path = os.path.join(self.__root_path, 'tmp')
    
    40
    +
    
    41
    +        os.makedirs(self.objects_path, exist_ok=True)
    
    42
    +        os.makedirs(self.temp_path, exist_ok=True)
    
    35 43
     
    
    36 44
         def has_blob(self, digest):
    
    37
    -        return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
    
    45
    +        return os.path.exists(self._get_object_path(digest))
    
    38 46
     
    
    39 47
         def get_blob(self, digest):
    
    40 48
             try:
    
    41
    -            return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
    
    49
    +            return open(self._get_object_path(digest), 'rb')
    
    42 50
             except FileNotFoundError:
    
    43 51
                 return None
    
    44 52
     
    
    45
    -    def begin_write(self, _digest):
    
    46
    -        return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
    
    53
    +    def begin_write(self, digest):
    
    54
    +        return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
    
    47 55
     
    
    48 56
         def commit_write(self, digest, write_session):
    
    49
    -        # Atomically move the temporary file into place.
    
    50
    -        path = self._path / (digest.hash + "_" + str(digest.size_bytes))
    
    51
    -        os.replace(write_session.name, str(path))
    
    57
    +        object_path = self._get_object_path(digest)
    
    58
    +
    
    52 59
             try:
    
    53
    -            write_session.close()
    
    54
    -        except FileNotFoundError:
    
    55
    -            # We moved the temporary file to a new location, so when Python
    
    56
    -            # tries to delete its old location, it'll fail.
    
    60
    +            os.makedirs(os.path.dirname(object_path), exist_ok=True)
    
    61
    +            os.link(write_session.name, object_path)
    
    62
    +        except FileExistsError:
    
    63
    +            # Object is already there!
    
    57 64
                 pass
    
    65
    +
    
    66
    +        write_session.close()
    
    67
    +
    
    68
    +    def _get_object_path(self, digest):
    
    69
    +        return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])

  • buildgrid/server/execution/instance.py
    ... ... @@ -21,8 +21,11 @@ An instance of the Remote Execution Service.
    21 21
     
    
    22 22
     import logging
    
    23 23
     
    
    24
    +import grpc
    
    25
    +
    
    24 26
     from buildgrid._exceptions import FailedPreconditionError, InvalidArgumentError
    
    25 27
     from buildgrid._protos.build.bazel.remote.execution.v2.remote_execution_pb2 import Action
    
    28
    +from buildgrid._protos.google.longrunning import operations_pb2
    
    26 29
     
    
    27 30
     from ..job import Job
    
    28 31
     
    
    ... ... @@ -71,7 +74,12 @@ class ExecutionInstance:
    71 74
     
    
    72 75
         def stream_operation_updates(self, message_queue, operation_name):
    
    73 76
             operation = message_queue.get()
    
    77
    +        last_operation = operation
    
    74 78
             while not operation.done:
    
    75 79
                 yield operation
    
    80
    +            last_operation = operation
    
    76 81
                 operation = message_queue.get()
    
    82
    +            if not isinstance(operation, operations_pb2.Operation):
    
    83
    +                message_queue.context_.cancel()
    
    84
    +                operation = last_operation
    
    77 85
             yield operation

  • buildgrid/server/execution/service.py
    ... ... @@ -52,6 +52,8 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    52 52
                 context.add_callback(partial(instance.unregister_message_client,
    
    53 53
                                              operation.name, message_queue))
    
    54 54
     
    
    55
    +            message_queue.context_ = context
    
    56
    +
    
    55 57
                 yield from instance.stream_operation_updates(message_queue,
    
    56 58
                                                              operation.name)
    
    57 59
     
    
    ... ... @@ -68,9 +70,10 @@ class ExecutionService(remote_execution_pb2_grpc.ExecutionServicer):
    68 70
                 yield operations_pb2.Operation()
    
    69 71
     
    
    70 72
         def WaitExecution(self, request, context):
    
    73
    +        print('Client reopened the operation stream!')
    
    74
    +
    
    71 75
             try:
    
    72 76
                 names = request.name.split("/")
    
    73
    -
    
    74 77
                 # Operation name should be in format:
    
    75 78
                 # {instance/name}/{operation_id}
    
    76 79
                 instance_name = ''.join(names[0:-1])
    

  • buildgrid/server/operations/service.py
    ... ... @@ -43,14 +43,16 @@ class OperationsService(operations_pb2_grpc.OperationsServicer):
    43 43
     
    
    44 44
         def GetOperation(self, request, context):
    
    45 45
             try:
    
    46
    -            name = request.name
    
    47
    -            operation_name = self._get_operation_name(name)
    
    46
    +            operation_name, job = request.name, None
    
    47
    +            for instance in self._instances.values():
    
    48
    +                if operation_name in instance._scheduler.jobs:
    
    49
    +                    job = instance._scheduler.jobs.get(operation_name)
    
    50
    +                    break
    
    48 51
     
    
    49
    -            instance = self._get_instance(name)
    
    52
    +            print('Connection drop simulation request received')
    
    50 53
     
    
    51
    -            operation = instance.get_operation(operation_name)
    
    52
    -            operation.name = name
    
    53
    -            return operation
    
    54
    +            for queue in job._operation_update_queues:
    
    55
    +                queue.put('drop')
    
    54 56
     
    
    55 57
             except InvalidArgumentError as e:
    
    56 58
                 self.logger.error(e)
    

  • docs/source/data/cas-example-server.conf
    1
    +server:
    
    2
    +  - !channel
    
    3
    +    port: 50051
    
    4
    +    insecure_mode: true
    
    5
    +
    
    6
    +instances:
    
    7
    +  - name: main
    
    8
    +
    
    9
    +    storages:
    
    10
    +      - !disk-storage &main-storage
    
    11
    +	path: !expand-path $HOME/cas
    
    12
    +
    
    13
    +    services:
    
    14
    +      - !cas
    
    15
    +	storage: *main-storage
    
    16
    +      - !bytestream
    
    17
    +	storage: *main-storage
    
    18
    +      - !reference-cache
    
    19
    +	storage: *main-storage
    
    20
    +	max_cached_refs: 512

  • docs/source/reference.rst
    1
    -
    
    2 1
     .. _reference:
    
    3 2
     
    
    4 3
     Reference
    
    ... ... @@ -11,3 +10,4 @@ This section contains BuildGrid API and CLI reference documentation.
    11 10
     
    
    12 11
        reference_api.rst
    
    13 12
        reference_cli.rst
    
    13
    +   reference_server_config.rst

  • docs/source/reference_cli.rst
    ... ... @@ -36,10 +36,10 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    36 36
     
    
    37 37
     ----
    
    38 38
     
    
    39
    -.. _invoking-bgd-bot-temp-directory:
    
    39
    +.. _invoking-bgd-bot-host-tools:
    
    40 40
     
    
    41
    -.. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
    
    42
    -   :prog: bgd bot temp-directory
    
    41
    +.. click:: buildgrid._app.commands.cmd_bot:run_host_tools
    
    42
    +   :prog: bgd bot host-tools
    
    43 43
     
    
    44 44
     ----
    
    45 45
     
    
    ... ... @@ -137,4 +137,4 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    137 137
     .. _invoking-bgd-server-start:
    
    138 138
     
    
    139 139
     .. click:: buildgrid._app.commands.cmd_server:start
    
    140
    -   :prog: bgd server start
    140
    +   :prog: bgd server start
    \ No newline at end of file

  • docs/source/reference_server_config.rst
    1
    +.. _parser:
    
    2
    +
    
    3
    +Server configuration reference
    
    4
    +==============================
    
    5
    +
    
    6
    +BuildGrid's server configuration. To be used with::
    
    7
    +
    
    8
    +  bgd server start server.conf
    
    9
    +
    
    10
    +.. automodule:: buildgrid._app.settings.parser
    
    11
    +    :members:
    
    12
    +    :undoc-members:
    
    13
    +    :show-inheritance:

  • docs/source/using.rst
    1
    -
    
    2 1
     .. _using:
    
    3 2
     
    
    4 3
     Using
    
    ... ... @@ -12,3 +11,5 @@ This section covers how to run an use the BuildGrid build service.
    12 11
        using_internal.rst
    
    13 12
        using_bazel.rst
    
    14 13
        using_buildstream.rst
    
    14
    +   using_recc.rst
    
    15
    +   using_cas_server.rst
    \ No newline at end of file

  • docs/source/using_bazel.rst
    ... ... @@ -94,12 +94,12 @@ has ``gcc`` installed, run:
    94 94
     
    
    95 95
     .. code-block:: sh
    
    96 96
     
    
    97
    -   bgd bot --remote=http://localhost:50051 --parent=main temp-directory
    
    97
    +   bgd bot --remote=http://localhost:50051 --parent=main host-tools
    
    98 98
     
    
    99 99
     The ``--remote`` option is used to specify the server location (running on the
    
    100 100
     same machine here, and listening to port 50051). The ``--parent`` option is used
    
    101
    -to specify the server instance you except the bot to be attached to. Refer to
    
    102
    -the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
    
    101
    +to specify the server instance you expect the bot to be attached to. Refer to
    
    102
    +the :ref:`CLI reference section <invoking-bgd-bot-host-tools>` for command
    
    103 103
     line interface details.
    
    104 104
     
    
    105 105
     The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
    
    ... ... @@ -120,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
    120 120
     
    
    121 121
        bazel build //main:hello-world
    
    122 122
     
    
    123
    -You can verify that the example has been successfully build by running the
    
    123
    +You can verify that the example has been successfully built by running the
    
    124 124
     generated executable. Simply invoke:
    
    125 125
     
    
    126 126
     .. code-block:: sh
    
    ... ... @@ -128,4 +128,4 @@ generated executable. Simply invoke:
    128 128
        ./bazel-bin/main/hello-world
    
    129 129
     
    
    130 130
     .. _bazel-examples: https://github.com/bazelbuild/examples
    
    131
    -.. _stage3 CPP example: https://github.com/bazelbuild/examples/tree/master/cpp-tutorial/stage3
    131
    +.. _stage3 CPP example: https://github.com/bazelbuild/examples/tree/master/cpp-tutorial/stage3
    \ No newline at end of file

  • docs/source/using_buildstream.rst
    ... ... @@ -149,7 +149,7 @@ that the ``buildbox`` tool is functional on your machine (refer to
    149 149
     
    
    150 150
     The ``--remote`` option is used to specify the server location (running on the
    
    151 151
     same machine here, and listening to port 50051). The ``--parent`` option is
    
    152
    -used to specify the server instance you except the bot to be attached to (empty
    
    152
    +used to specify the server instance you expect the bot to be attached to (empty
    
    153 153
     here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
    
    154 154
     and respectively specify the sandbox mount point and local CAS cache locations.
    
    155 155
     Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
    
    ... ... @@ -178,7 +178,7 @@ You can finally have BuildStream to build the example project by running:
    178 178
     
    
    179 179
        bst build hello.bst
    
    180 180
     
    
    181
    -You can verify that the example has been successfully build by running the
    
    181
    +You can verify that the example has been successfully built by running the
    
    182 182
     generated executable. Simply invoke:
    
    183 183
     
    
    184 184
     .. code-block:: sh
    

  • docs/source/using_cas_server.rst
    1
    +.. _cas-server:
    
    2
    +
    
    3
    +CAS server
    
    4
    +==========
    
    5
    +
    
    6
    +It is possible to configure BuildGrid with just a Content Addressable Storage service.
    
    7
    +
    
    8
    +.. note::
    
    9
    +
    
    10
    +   This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
    
    11
    +
    
    12
    +.. _cas-configuration:
    
    13
    +
    
    14
    +Configuration
    
    15
    +-------------
    
    16
    +
    
    17
    +Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
    
    18
    +
    
    19
    +.. literalinclude:: ./data/cas-example-server.conf
    
    20
    +   :language: yaml
    
    21
    +
    
    22
    +.. hint::
    
    23
    +
    
    24
    +   Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
    
    25
    +
    
    26
    +This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
    
    27
    +
    
    28
    +.. code-block:: sh
    
    29
    +
    
    30
    +   bgd server start example.conf
    
    31
    +
    
    32
    +The server should now be available to use.
    
    33
    +
    
    34
    +.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
    
    35
    +.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto

  • docs/source/using_internal.rst
    1
    +
    
    1 2
     .. _internal-client:
    
    2 3
     
    
    3 4
     Internal client
    
    ... ... @@ -83,7 +84,7 @@ Start the following bot session:
    83 84
     
    
    84 85
     .. code-block:: sh
    
    85 86
     
    
    86
    -   bgd bot temp-directory
    
    87
    +   bgd bot host-tools
    
    87 88
     
    
    88 89
     Upload the directory containing the C file:
    
    89 90
     
    
    ... ... @@ -101,4 +102,4 @@ to run on the bot:
    101 102
        bgd execute command --output-file hello True /path/to/test-buildgrid -- gcc -Wall hello.c -o hello
    
    102 103
     
    
    103 104
     The resulting executeable should have returned to a new directory called
    
    104
    -``testing``.
    105
    +``testing``.
    \ No newline at end of file

  • docs/source/using_recc.rst
    1
    +
    
    2
    +.. _recc-client:
    
    3
    +
    
    4
    +RECC client
    
    5
    +===========
    
    6
    +
    
    7
    +`RECC`_ is the *Remote Execution Caching Compiler*, an open source build tool
    
    8
    +that wraps compiler command calls and forwards them to a remote build execution
    
    9
    +service using the remote execution API (REAPI) v2.
    
    10
    +
    
    11
    +.. note::
    
    12
    +
    
    13
    +   There is no stable release of RECC yet. You'll have to `install it from
    
    14
    +   sources`_.
    
    15
    +
    
    16
    +.. _RECC: https://gitlab.com/bloomberg/recc
    
    17
    +.. _install it from sources: https://gitlab.com/bloomberg/recc#installing-dependencies
    
    18
    +
    
    19
    +
    
    20
    +.. _recc-configuration:
    
    21
    +
    
    22
    +Configuration
    
    23
    +-------------
    
    24
    +
    
    25
    +RECC reads the configuration from its execution environment. You can get a
    
    26
    +complete list of environment variables it accepts by running:
    
    27
    +
    
    28
    +.. code-block:: sh
    
    29
    +
    
    30
    +   recc --help
    
    31
    +
    
    32
    +The variables are prefixed with ``RECC_``. The most important ones for remote
    
    33
    +execution are:
    
    34
    +
    
    35
    +- ``RECC_SERVER``: URI of the remote execution server.
    
    36
    +- ``RECC_CAS_SERVER``: URI of the CAS server, defaults to ``RECC_SERVER``.
    
    37
    +- ``RECC_INSTANCE``: name of the remote execution instance.
    
    38
    +
    
    39
    +.. hint::
    
    40
    +
    
    41
    +   ``RECC_VERBOSE=1`` can be set in order to enable verbose output.
    
    42
    +
    
    43
    +As an example, in order to forward compile commands to the ``main`` instance of
    
    44
    +the remote execution server available at ``controller.grid.build`` on port
    
    45
    +``50051`` you should export:
    
    46
    +
    
    47
    +.. code-block:: sh
    
    48
    +
    
    49
    +   export RECC_SERVER=controller.grid.build:50051
    
    50
    +   export RECC_INSTANCE=main
    
    51
    +
    
    52
    +
    
    53
    +.. _recc-example:
    
    54
    +
    
    55
    +Example build
    
    56
    +-------------
    
    57
    +
    
    58
    +RECC can be use with any existing software package respecting `GNU make common
    
    59
    +variables`_ like ``CC`` for the C compiler or ``CXX`` for the C++ compiler.
    
    60
    +We'll focus here on instructions on how to build the `GNU Hello`_ example
    
    61
    +program using RECC and BuildGrid on your local machine.
    
    62
    +
    
    63
    +First, you need to download the hello source package:
    
    64
    +
    
    65
    +.. code-block:: sh
    
    66
    +
    
    67
    +   wget https://ftp.gnu.org/gnu/hello/hello-2.10.tar.gz
    
    68
    +
    
    69
    +Next, unpack it and change the current directory to the source root:
    
    70
    +
    
    71
    +.. code-block:: sh
    
    72
    +
    
    73
    +   tar xvf hello-2.10.tar.gz
    
    74
    +   cd hello-2.10
    
    75
    +
    
    76
    +.. hint::
    
    77
    +
    
    78
    +   All the commands in the instructions below are expected to be executed from
    
    79
    +   that root source directory (the GNU Hello project's root directory).
    
    80
    +
    
    81
    +Before trying to build the hello example program, you'll have to setup and run a
    
    82
    +BuildGrid server and bot. A minimal server's configuration is given below, paste
    
    83
    +it in a ``server.conf`` file in the root directory:
    
    84
    +
    
    85
    +.. literalinclude:: ./data/bazel-example-server.conf
    
    86
    +   :language: yaml
    
    87
    +
    
    88
    +This defines a single ``main`` server instance implementing a
    
    89
    +``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
    
    90
    +``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
    
    91
    +You can then start the BuildGrid server daemon using that configuration by
    
    92
    +running:
    
    93
    +
    
    94
    +.. code-block:: sh
    
    95
    +
    
    96
    +   bgd server start server.conf
    
    97
    +
    
    98
    +In order to perform the actual build work, you need to attach a worker bot to
    
    99
    +that server for that ``main`` instance. RECC comes with its own ``reccworker``
    
    100
    +bot implementation. However, BuildGrid's host-tools based bot should be enough
    
    101
    +to build the hello example program. Once you've make sure that your machine has
    
    102
    +``gcc`` installed, run:
    
    103
    +
    
    104
    +.. code-block:: sh
    
    105
    +
    
    106
    +   bgd bot --remote=http://localhost:50051 --parent=main host-tools
    
    107
    +
    
    108
    +The ``--remote`` option is used to specify the server location (running on the
    
    109
    +same machine here, and listening to port 50051). The ``--parent`` option is used
    
    110
    +to specify the server instance you expect the bot to be attached to. Refer to
    
    111
    +the :ref:`CLI reference section <invoking-bgd-bot-host-tools>` for command
    
    112
    +line interface details.
    
    113
    +
    
    114
    +The BuildGrid server is now ready to accept jobs and execute them. RECC's
    
    115
    +:ref:`configuration <bazel-configuration>` needs to be defined as environment
    
    116
    +variables. Define minimal configuration by running:
    
    117
    +
    
    118
    +.. code-block:: sh
    
    119
    +
    
    120
    +   export RECC_SERVER=localhost:50051
    
    121
    +   export RECC_INSTANCE=main
    
    122
    +
    
    123
    +This points RECC to the ``main`` remote execution server instance at
    
    124
    +``localhost:50051``.
    
    125
    +
    
    126
    +GNU Hello is using `The Autotools`_ as a build system, so first, you need to
    
    127
    +configure your build. Run:
    
    128
    +
    
    129
    +.. code-block:: sh
    
    130
    +
    
    131
    +   ./configure
    
    132
    +
    
    133
    +You can finally build the hello example program, using RECC by running:
    
    134
    +
    
    135
    +.. code-block:: sh
    
    136
    +
    
    137
    +   make CC="recc cc"
    
    138
    +
    
    139
    +You can verify that the example program has been successfully built by running
    
    140
    +the generated executable. Simply invoke:
    
    141
    +
    
    142
    +.. code-block:: sh
    
    143
    +
    
    144
    +   ./hello
    
    145
    +
    
    146
    +.. _GNU make common variables: https://www.gnu.org/software/make/manual/html_node/Implicit-Variables.html
    
    147
    +.. _GNU Hello: https://www.gnu.org/software/hello
    
    148
    +.. _The Autotools: https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html
    \ No newline at end of file

  • setup.py
    ... ... @@ -97,7 +97,8 @@ tests_require = [
    97 97
     ]
    
    98 98
     
    
    99 99
     docs_require = [
    
    100
    -    'sphinx',
    
    100
    +    # rtd-theme broken in Sphinx >= 1.8, this breaks search functionality.
    
    101
    +    'sphinx == 1.7.8',
    
    101 102
         'sphinx-click',
    
    102 103
         'sphinx-rtd-theme',
    
    103 104
         'sphinxcontrib-apidoc',
    

  • tests/server_instance.py
    ... ... @@ -44,8 +44,8 @@ instances:
    44 44
           The main server
    
    45 45
     
    
    46 46
         storages:
    
    47
    -        - !disk-storage &main-storage
    
    48
    -          path: ~/cas/
    
    47
    +        - !lru-storage &main-storage
    
    48
    +          size: 256mb
    
    49 49
     
    
    50 50
         services:
    
    51 51
           - !action-cache &main-action
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]