Martin Blanchard pushed to branch finn/remove-disk-storage-from-tests at BuildGrid / buildgrid
Commits:
-
d6e6e64e
by Finn at 2018-10-03T09:36:53Z
-
40b2e514
by Finn at 2018-10-03T09:46:35Z
-
b8003277
by Martin Blanchard at 2018-10-03T09:54:47Z
-
545e9b10
by Finn at 2018-10-03T10:01:23Z
8 changed files:
- buildgrid/_app/settings/parser.py
- + docs/source/data/cas-example-server.conf
- docs/source/reference.rst
- + docs/source/reference_server_config.rst
- docs/source/using.rst
- + docs/source/using_cas_server.rst
- + docs/source/using_recc.rst
- tests/server_instance.py
Changes:
... | ... | @@ -35,6 +35,9 @@ from ..cli import Context |
35 | 35 |
|
36 | 36 |
|
37 | 37 |
class YamlFactory(yaml.YAMLObject):
|
38 |
+ """ Base class for contructing maps or scalars from tags.
|
|
39 |
+ """
|
|
40 |
+ |
|
38 | 41 |
@classmethod
|
39 | 42 |
def from_yaml(cls, loader, node):
|
40 | 43 |
if isinstance(node, yaml.ScalarNode):
|
... | ... | @@ -47,6 +50,21 @@ class YamlFactory(yaml.YAMLObject): |
47 | 50 |
|
48 | 51 |
|
49 | 52 |
class Channel(YamlFactory):
|
53 |
+ """Creates a GRPC channel.
|
|
54 |
+ |
|
55 |
+ The :class:`Channel` class returns a `grpc.Channel` and is generated from the tag ``!channel``.
|
|
56 |
+ Creates either a secure or insecure channel.
|
|
57 |
+ |
|
58 |
+ Args:
|
|
59 |
+ port (int): A port for the channel.
|
|
60 |
+ insecure_mode (bool): If ``True``, generates an insecure channel, even if there are
|
|
61 |
+ credentials. Defaults to ``True``.
|
|
62 |
+ credentials (dict, optional): A dictionary in the form::
|
|
63 |
+ |
|
64 |
+ tls-server-key: /path/to/server-key
|
|
65 |
+ tls-server-cert: /path/to/server-cert
|
|
66 |
+ tls-client-certs: /path/to/client-certs
|
|
67 |
+ """
|
|
50 | 68 |
|
51 | 69 |
yaml_tag = u'!channel'
|
52 | 70 |
|
... | ... | @@ -69,6 +87,13 @@ class Channel(YamlFactory): |
69 | 87 |
|
70 | 88 |
|
71 | 89 |
class ExpandPath(YamlFactory):
|
90 |
+ """Returns a string of the user's path after expansion.
|
|
91 |
+ |
|
92 |
+ The :class:`ExpandPath` class returns a string and is generated from the tag ``!expand-path``.
|
|
93 |
+ |
|
94 |
+ Args:
|
|
95 |
+ path (str): Can be used with strings such as: ``~/dir/to/something`` or ``$HOME/certs``
|
|
96 |
+ """
|
|
72 | 97 |
|
73 | 98 |
yaml_tag = u'!expand-path'
|
74 | 99 |
|
... | ... | @@ -79,14 +104,30 @@ class ExpandPath(YamlFactory): |
79 | 104 |
|
80 | 105 |
|
81 | 106 |
class Disk(YamlFactory):
|
107 |
+ """Generates :class:`buildgrid.server.cas.storage.disk.DiskStorage` using the tag ``!disk-storage``.
|
|
108 |
+ |
|
109 |
+ Args:
|
|
110 |
+ path (str): Path to directory to storage.
|
|
111 |
+ |
|
112 |
+ """
|
|
82 | 113 |
|
83 | 114 |
yaml_tag = u'!disk-storage'
|
84 | 115 |
|
85 | 116 |
def __new__(cls, path):
|
117 |
+ """Creates a new disk
|
|
118 |
+ |
|
119 |
+ Args:
|
|
120 |
+ path (str): Some path
|
|
121 |
+ """
|
|
86 | 122 |
return DiskStorage(path)
|
87 | 123 |
|
88 | 124 |
|
89 | 125 |
class LRU(YamlFactory):
|
126 |
+ """Generates :class:`buildgrid.server.cas.storage.lru_memory_cache.LRUMemoryCache` using the tag ``!lru-storage``.
|
|
127 |
+ |
|
128 |
+ Args:
|
|
129 |
+ size (int): Size e.g ``10kb``. Size parsed with :meth:`buildgrid._app.settings.parser._parse_size`.
|
|
130 |
+ """
|
|
90 | 131 |
|
91 | 132 |
yaml_tag = u'!lru-storage'
|
92 | 133 |
|
... | ... | @@ -95,6 +136,12 @@ class LRU(YamlFactory): |
95 | 136 |
|
96 | 137 |
|
97 | 138 |
class S3(YamlFactory):
|
139 |
+ """Generates :class:`buildgrid.server.cas.storage.s3.S3Storage` using the tag ``!s3-storage``.
|
|
140 |
+ |
|
141 |
+ Args:
|
|
142 |
+ bucket (str): Name of bucket
|
|
143 |
+ endpoint (str): URL of endpoint.
|
|
144 |
+ """
|
|
98 | 145 |
|
99 | 146 |
yaml_tag = u'!s3-storage'
|
100 | 147 |
|
... | ... | @@ -103,6 +150,18 @@ class S3(YamlFactory): |
103 | 150 |
|
104 | 151 |
|
105 | 152 |
class Remote(YamlFactory):
|
153 |
+ """Generates :class:`buildgrid.server.cas.storage.remote.RemoteStorage`
|
|
154 |
+ using the tag ``!remote-storage``.
|
|
155 |
+ |
|
156 |
+ Args:
|
|
157 |
+ url (str): URL to remote storage. If used with ``https``, needs credentials.
|
|
158 |
+ instance_name (str): Instance of the remote to connect to.
|
|
159 |
+ credentials (dict, optional): A dictionary in the form::
|
|
160 |
+ |
|
161 |
+ tls-client-key: /path/to/client-key
|
|
162 |
+ tls-client-cert: /path/to/client-cert
|
|
163 |
+ tls-server-cert: /path/to/server-cert
|
|
164 |
+ """
|
|
106 | 165 |
|
107 | 166 |
yaml_tag = u'!remote-storage'
|
108 | 167 |
|
... | ... | @@ -144,6 +203,18 @@ class Remote(YamlFactory): |
144 | 203 |
|
145 | 204 |
|
146 | 205 |
class WithCache(YamlFactory):
|
206 |
+ """Generates :class:`buildgrid.server.cas.storage.with_cache.WithCacheStorage`
|
|
207 |
+ using the tag ``!with-cache-storage``.
|
|
208 |
+ |
|
209 |
+ Args:
|
|
210 |
+ url (str): URL to remote storage. If used with ``https``, needs credentials.
|
|
211 |
+ instance_name (str): Instance of the remote to connect to.
|
|
212 |
+ credentials (dict, optional): A dictionary in the form::
|
|
213 |
+ |
|
214 |
+ tls-client-key: /path/to/certs
|
|
215 |
+ tls-client-cert: /path/to/certs
|
|
216 |
+ tls-server-cert: /path/to/certs
|
|
217 |
+ """
|
|
147 | 218 |
|
148 | 219 |
yaml_tag = u'!with-cache-storage'
|
149 | 220 |
|
... | ... | @@ -152,6 +223,13 @@ class WithCache(YamlFactory): |
152 | 223 |
|
153 | 224 |
|
154 | 225 |
class Execution(YamlFactory):
|
226 |
+ """Generates :class:`buildgrid.server.execution.service.ExecutionService`
|
|
227 |
+ using the tag ``!execution``.
|
|
228 |
+ |
|
229 |
+ Args:
|
|
230 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
231 |
+ action_cache(:class:`Action`): Instance of action cache to use.
|
|
232 |
+ """
|
|
155 | 233 |
|
156 | 234 |
yaml_tag = u'!execution'
|
157 | 235 |
|
... | ... | @@ -160,6 +238,14 @@ class Execution(YamlFactory): |
160 | 238 |
|
161 | 239 |
|
162 | 240 |
class Action(YamlFactory):
|
241 |
+ """Generates :class:`buildgrid.server.actioncache.service.ActionCacheService`
|
|
242 |
+ using the tag ``!action-cache``.
|
|
243 |
+ |
|
244 |
+ Args:
|
|
245 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
246 |
+ max_cached_refs(int): Max number of cached actions.
|
|
247 |
+ allow_updates(bool): Allow updates pushed to CAS. Defaults to ``True``.
|
|
248 |
+ """
|
|
163 | 249 |
|
164 | 250 |
yaml_tag = u'!action-cache'
|
165 | 251 |
|
... | ... | @@ -168,6 +254,14 @@ class Action(YamlFactory): |
168 | 254 |
|
169 | 255 |
|
170 | 256 |
class Reference(YamlFactory):
|
257 |
+ """Generates :class:`buildgrid.server.referencestorage.service.ReferenceStorageService`
|
|
258 |
+ using the tag ``!reference-cache``.
|
|
259 |
+ |
|
260 |
+ Args:
|
|
261 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
262 |
+ max_cached_refs(int): Max number of cached actions.
|
|
263 |
+ allow_updates(bool): Allow updates pushed to CAS. Defauled to ``True``.
|
|
264 |
+ """
|
|
171 | 265 |
|
172 | 266 |
yaml_tag = u'!reference-cache'
|
173 | 267 |
|
... | ... | @@ -176,6 +270,12 @@ class Reference(YamlFactory): |
176 | 270 |
|
177 | 271 |
|
178 | 272 |
class CAS(YamlFactory):
|
273 |
+ """Generates :class:`buildgrid.server.cas.service.ContentAddressableStorageService`
|
|
274 |
+ using the tag ``!cas``.
|
|
275 |
+ |
|
276 |
+ Args:
|
|
277 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
278 |
+ """
|
|
179 | 279 |
|
180 | 280 |
yaml_tag = u'!cas'
|
181 | 281 |
|
... | ... | @@ -184,6 +284,12 @@ class CAS(YamlFactory): |
184 | 284 |
|
185 | 285 |
|
186 | 286 |
class ByteStream(YamlFactory):
|
287 |
+ """Generates :class:`buildgrid.server.cas.service.ByteStreamService`
|
|
288 |
+ using the tag ``!bytestream``.
|
|
289 |
+ |
|
290 |
+ Args:
|
|
291 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
292 |
+ """
|
|
187 | 293 |
|
188 | 294 |
yaml_tag = u'!bytestream'
|
189 | 295 |
|
1 |
+server:
|
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure_mode: true
|
|
5 |
+ |
|
6 |
+instances:
|
|
7 |
+ - name: main
|
|
8 |
+ |
|
9 |
+ storages:
|
|
10 |
+ - !disk-storage &main-storage
|
|
11 |
+ path: !expand-path $HOME/cas
|
|
12 |
+ |
|
13 |
+ services:
|
|
14 |
+ - !cas
|
|
15 |
+ storage: *main-storage
|
|
16 |
+ - !bytestream
|
|
17 |
+ storage: *main-storage
|
|
18 |
+ - !reference-cache
|
|
19 |
+ storage: *main-storage
|
|
20 |
+ max_cached_refs: 512
|
1 |
- |
|
2 | 1 |
.. _reference:
|
3 | 2 |
|
4 | 3 |
Reference
|
... | ... | @@ -11,3 +10,4 @@ This section contains BuildGrid API and CLI reference documentation. |
11 | 10 |
|
12 | 11 |
reference_api.rst
|
13 | 12 |
reference_cli.rst
|
13 |
+ reference_server_config.rst
|
1 |
+.. _parser:
|
|
2 |
+ |
|
3 |
+Server configuration reference
|
|
4 |
+==============================
|
|
5 |
+ |
|
6 |
+BuildGrid's server configuration. To be used with::
|
|
7 |
+ |
|
8 |
+ bgd server start server.conf
|
|
9 |
+ |
|
10 |
+.. automodule:: buildgrid._app.settings.parser
|
|
11 |
+ :members:
|
|
12 |
+ :undoc-members:
|
|
13 |
+ :show-inheritance:
|
1 |
- |
|
2 | 1 |
.. _using:
|
3 | 2 |
|
4 | 3 |
Using
|
... | ... | @@ -12,3 +11,5 @@ This section covers how to run an use the BuildGrid build service. |
12 | 11 |
using_internal.rst
|
13 | 12 |
using_bazel.rst
|
14 | 13 |
using_buildstream.rst
|
14 |
+ using_recc.rst
|
|
15 |
+ using_cas_server.rst
|
|
\ No newline at end of file |
1 |
+.. _cas-server:
|
|
2 |
+ |
|
3 |
+CAS server
|
|
4 |
+==========
|
|
5 |
+ |
|
6 |
+It is possible to configure BuildGrid with just a Content Addressable Storage service.
|
|
7 |
+ |
|
8 |
+.. note::
|
|
9 |
+ |
|
10 |
+ This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
|
|
11 |
+ |
|
12 |
+.. _cas-configuration:
|
|
13 |
+ |
|
14 |
+Configuration
|
|
15 |
+-------------
|
|
16 |
+ |
|
17 |
+Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
|
|
18 |
+ |
|
19 |
+.. literalinclude:: ./data/cas-example-server.conf
|
|
20 |
+ :language: yaml
|
|
21 |
+ |
|
22 |
+.. hint::
|
|
23 |
+ |
|
24 |
+ Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
|
|
25 |
+ |
|
26 |
+This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
|
|
27 |
+ |
|
28 |
+.. code-block:: sh
|
|
29 |
+ |
|
30 |
+ bgd server start example.conf
|
|
31 |
+ |
|
32 |
+The server should now be available to use.
|
|
33 |
+ |
|
34 |
+.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
|
|
35 |
+.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto
|
1 |
+ |
|
2 |
+.. _recc-client:
|
|
3 |
+ |
|
4 |
+RECC client
|
|
5 |
+===========
|
|
6 |
+ |
|
7 |
+`RECC`_ is the *Remote Execution Caching Compiler*, an open source build tool
|
|
8 |
+that wraps compiler command calls and forwards them to a remote build execution
|
|
9 |
+service using the remote execution API (REAPI) v2.
|
|
10 |
+ |
|
11 |
+.. note::
|
|
12 |
+ |
|
13 |
+ There is no stable release of RECC yet. You'll have to `install it from
|
|
14 |
+ sources`_.
|
|
15 |
+ |
|
16 |
+.. _RECC: https://gitlab.com/bloomberg/recc
|
|
17 |
+.. _install it from sources: https://gitlab.com/bloomberg/recc#installing-dependencies
|
|
18 |
+ |
|
19 |
+ |
|
20 |
+.. _recc-configuration:
|
|
21 |
+ |
|
22 |
+Configuration
|
|
23 |
+-------------
|
|
24 |
+ |
|
25 |
+RECC reads the configuration from its execution environment. You can get a
|
|
26 |
+complete list of environment variables it accepts by running:
|
|
27 |
+ |
|
28 |
+.. code-block:: sh
|
|
29 |
+ |
|
30 |
+ recc --help
|
|
31 |
+ |
|
32 |
+The variables are prefixed with ``RECC_``. The most important ones for remote
|
|
33 |
+execution are:
|
|
34 |
+ |
|
35 |
+- ``RECC_SERVER``: URI of the remote execution server.
|
|
36 |
+- ``RECC_CAS_SERVER``: URI of the CAS server, defaults to ``RECC_SERVER``.
|
|
37 |
+- ``RECC_INSTANCE``: name of the remote execution instance.
|
|
38 |
+ |
|
39 |
+.. hint::
|
|
40 |
+ |
|
41 |
+ ``RECC_VERBOSE=1`` can be set in order to enable verbose output.
|
|
42 |
+ |
|
43 |
+As an example, in order to forward compile commands to the ``main`` instance of
|
|
44 |
+the remote execution server available at ``controller.grid.build`` on port
|
|
45 |
+``50051`` you should export:
|
|
46 |
+ |
|
47 |
+.. code-block:: sh
|
|
48 |
+ |
|
49 |
+ export RECC_SERVER=controller.grid.build:50051
|
|
50 |
+ export RECC_INSTANCE=main
|
|
51 |
+ |
|
52 |
+ |
|
53 |
+.. _recc-example:
|
|
54 |
+ |
|
55 |
+Example build
|
|
56 |
+-------------
|
|
57 |
+ |
|
58 |
+RECC can be use with any existing software package respecting `GNU make common
|
|
59 |
+variables`_ like ``CC`` for the C compiler or ``CXX`` for the C++ compiler.
|
|
60 |
+We'll focus here on instructions on how to build the `GNU Hello`_ example
|
|
61 |
+program using RECC and BuildGrid on your local machine.
|
|
62 |
+ |
|
63 |
+First, you need to download the hello source package:
|
|
64 |
+ |
|
65 |
+.. code-block:: sh
|
|
66 |
+ |
|
67 |
+ wget https://ftp.gnu.org/gnu/hello/hello-2.10.tar.gz
|
|
68 |
+ |
|
69 |
+Next, unpack it and change the current directory to the source root:
|
|
70 |
+ |
|
71 |
+.. code-block:: sh
|
|
72 |
+ |
|
73 |
+ tar xvf hello-2.10.tar.gz
|
|
74 |
+ cd hello-2.10
|
|
75 |
+ |
|
76 |
+.. hint::
|
|
77 |
+ |
|
78 |
+ All the commands in the instructions below are expected to be executed from
|
|
79 |
+ that root source directory (the GNU Hello project's root directory).
|
|
80 |
+ |
|
81 |
+Before trying to build the hello example program, you'll have to setup and run a
|
|
82 |
+BuildGrid server and bot. A minimal server's configuration is given below, paste
|
|
83 |
+it in a ``server.conf`` file in the root directory:
|
|
84 |
+ |
|
85 |
+.. literalinclude:: ./data/bazel-example-server.conf
|
|
86 |
+ :language: yaml
|
|
87 |
+ |
|
88 |
+This defines a single ``main`` server instance implementing a
|
|
89 |
+``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
|
|
90 |
+``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
|
|
91 |
+You can then start the BuildGrid server daemon using that configuration by
|
|
92 |
+running:
|
|
93 |
+ |
|
94 |
+.. code-block:: sh
|
|
95 |
+ |
|
96 |
+ bgd server start server.conf
|
|
97 |
+ |
|
98 |
+In order to perform the actual build work, you need to attach a worker bot to
|
|
99 |
+that server for that ``main`` instance. RECC comes with its own ``reccworker``
|
|
100 |
+bot implementation. However, BuildGrid's host-tools based bot should be enough
|
|
101 |
+to build the hello example program. Once you've make sure that your machine has
|
|
102 |
+``gcc`` installed, run:
|
|
103 |
+ |
|
104 |
+.. code-block:: sh
|
|
105 |
+ |
|
106 |
+ bgd bot --remote=http://localhost:50051 --parent=main temp-directory
|
|
107 |
+ |
|
108 |
+The ``--remote`` option is used to specify the server location (running on the
|
|
109 |
+same machine here, and listening to port 50051). The ``--parent`` option is used
|
|
110 |
+to specify the server instance you expect the bot to be attached to. Refer to
|
|
111 |
+the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
|
|
112 |
+line interface details.
|
|
113 |
+ |
|
114 |
+The BuildGrid server is now ready to accept jobs and execute them. RECC's
|
|
115 |
+:ref:`configuration <bazel-configuration>` needs to be defined as environment
|
|
116 |
+variables. Define minimal configuration by running:
|
|
117 |
+ |
|
118 |
+.. code-block:: sh
|
|
119 |
+ |
|
120 |
+ export RECC_SERVER=localhost:50051
|
|
121 |
+ export RECC_INSTANCE=main
|
|
122 |
+ |
|
123 |
+This points RECC to the ``main`` remote execution server instance at
|
|
124 |
+``localhost:50051``.
|
|
125 |
+ |
|
126 |
+GNU Hello is using `The Autotools`_ as a build system, so first, you need to
|
|
127 |
+configure your build. Run:
|
|
128 |
+ |
|
129 |
+.. code-block:: sh
|
|
130 |
+ |
|
131 |
+ ./configure
|
|
132 |
+ |
|
133 |
+You can finally build the hello example program, using RECC by running:
|
|
134 |
+ |
|
135 |
+.. code-block:: sh
|
|
136 |
+ |
|
137 |
+ make CC="recc cc"
|
|
138 |
+ |
|
139 |
+You can verify that the example program has been successfully built by running
|
|
140 |
+the generated executable. Simply invoke:
|
|
141 |
+ |
|
142 |
+.. code-block:: sh
|
|
143 |
+ |
|
144 |
+ ./hello
|
|
145 |
+ |
|
146 |
+.. _GNU make common variables: https://www.gnu.org/software/make/manual/html_node/Implicit-Variables.html
|
|
147 |
+.. _GNU Hello: https://www.gnu.org/software/hello
|
|
148 |
+.. _The Autotools: https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html
|
|
\ No newline at end of file |
... | ... | @@ -44,8 +44,8 @@ instances: |
44 | 44 |
The main server
|
45 | 45 |
|
46 | 46 |
storages:
|
47 |
- - !disk-storage &main-storage
|
|
48 |
- path: ~/cas/
|
|
47 |
+ - !lru-storage &main-storage
|
|
48 |
+ size: 256mb
|
|
49 | 49 |
|
50 | 50 |
services:
|
51 | 51 |
- !action-cache &main-action
|