Jürg Billeter pushed to branch juerg/buildbox-env at BuildGrid / buildgrid
Commits:
-
9401c87d
by Finn at 2018-10-02T16:33:03Z
-
51f661fd
by Jürg Billeter at 2018-10-03T07:50:34Z
-
febe2ced
by Martin Blanchard at 2018-10-03T08:08:56Z
-
e80b6503
by Martin Blanchard at 2018-10-03T08:08:56Z
-
a305c9a1
by Finn at 2018-10-03T09:22:30Z
-
d6e6e64e
by Finn at 2018-10-03T09:36:53Z
-
40b2e514
by Finn at 2018-10-03T09:46:35Z
-
b8003277
by Martin Blanchard at 2018-10-03T09:54:47Z
-
545e9b10
by Finn at 2018-10-03T10:01:23Z
-
321c34d8
by Martin Blanchard at 2018-10-03T10:11:17Z
-
a539c4e9
by Finn at 2018-10-03T10:49:00Z
-
7cc13d06
by Finn at 2018-10-03T10:49:00Z
-
e59fb607
by Finn at 2018-10-03T13:10:21Z
-
23f0fed1
by Jürg Billeter at 2018-10-03T13:16:10Z
21 changed files:
- buildgrid/_app/bots/buildbox.py
- buildgrid/_app/bots/temp_directory.py → buildgrid/_app/bots/host.py
- buildgrid/_app/commands/cmd_bot.py
- buildgrid/_app/settings/parser.py
- buildgrid/client/cas.py
- buildgrid/server/actioncache/service.py
- buildgrid/server/cas/storage/disk.py
- buildgrid/server/job.py
- buildgrid/server/scheduler.py
- + docs/source/data/cas-example-server.conf
- docs/source/reference.rst
- docs/source/reference_cli.rst
- + docs/source/reference_server_config.rst
- docs/source/using.rst
- docs/source/using_bazel.rst
- docs/source/using_buildstream.rst
- + docs/source/using_cas_server.rst
- docs/source/using_internal.rst
- + docs/source/using_recc.rst
- setup.py
- tests/server_instance.py
Changes:
... | ... | @@ -46,11 +46,6 @@ def work_buildbox(context, lease): |
46 | 46 |
command = downloader.get_message(action.command_digest,
|
47 | 47 |
remote_execution_pb2.Command())
|
48 | 48 |
|
49 |
- environment = {}
|
|
50 |
- for variable in command.environment_variables:
|
|
51 |
- if variable.name not in ['PWD']:
|
|
52 |
- environment[variable.name] = variable.value
|
|
53 |
- |
|
54 | 49 |
if command.working_directory:
|
55 | 50 |
working_directory = command.working_directory
|
56 | 51 |
else:
|
... | ... | @@ -82,6 +77,12 @@ def work_buildbox(context, lease): |
82 | 77 |
if context.cas_server_cert:
|
83 | 78 |
command_line.append('--server-cert={}'.format(context.cas_server_cert))
|
84 | 79 |
|
80 |
+ command_line.append('--clearenv')
|
|
81 |
+ for variable in command.environment_variables:
|
|
82 |
+ command_line.append('--setenv')
|
|
83 |
+ command_line.append(variable.name)
|
|
84 |
+ command_line.append(variable.value)
|
|
85 |
+ |
|
85 | 86 |
command_line.append(context.fuse_dir)
|
86 | 87 |
command_line.extend(command.arguments)
|
87 | 88 |
|
... | ... | @@ -24,7 +24,7 @@ from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_p |
24 | 24 |
from buildgrid.utils import output_file_maker, output_directory_maker
|
25 | 25 |
|
26 | 26 |
|
27 |
-def work_temp_directory(context, lease):
|
|
27 |
+def work_host_tools(context, lease):
|
|
28 | 28 |
"""Executes a lease for a build action, using host tools.
|
29 | 29 |
"""
|
30 | 30 |
|
... | ... | @@ -31,7 +31,7 @@ import grpc |
31 | 31 |
from buildgrid.bot import bot, bot_interface
|
32 | 32 |
from buildgrid.bot.bot_session import BotSession, Device, Worker
|
33 | 33 |
|
34 |
-from ..bots import buildbox, dummy, temp_directory
|
|
34 |
+from ..bots import buildbox, dummy, host
|
|
35 | 35 |
from ..cli import pass_context
|
36 | 36 |
|
37 | 37 |
|
... | ... | @@ -52,16 +52,19 @@ from ..cli import pass_context |
52 | 52 |
help="Public CAS client certificate for TLS (PEM-encoded)")
|
53 | 53 |
@click.option('--cas-server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
|
54 | 54 |
help="Public CAS server certificate for TLS (PEM-encoded)")
|
55 |
+@click.option('--update-period', type=click.FLOAT, default=0.5, show_default=True,
|
|
56 |
+ help="Time period for bot updates to the server in seconds.")
|
|
55 | 57 |
@click.option('--parent', type=click.STRING, default='main', show_default=True,
|
56 | 58 |
help="Targeted farm resource.")
|
57 | 59 |
@pass_context
|
58 |
-def cli(context, parent, remote, client_key, client_cert, server_cert,
|
|
60 |
+def cli(context, parent, update_period, remote, client_key, client_cert, server_cert,
|
|
59 | 61 |
remote_cas, cas_client_key, cas_client_cert, cas_server_cert):
|
60 | 62 |
# Setup the remote execution server channel:
|
61 | 63 |
url = urlparse(remote)
|
62 | 64 |
|
63 | 65 |
context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
|
64 | 66 |
context.remote_url = remote
|
67 |
+ context.update_period = update_period
|
|
65 | 68 |
context.parent = parent
|
66 | 69 |
|
67 | 70 |
if url.scheme == 'http':
|
... | ... | @@ -135,26 +138,26 @@ def cli(context, parent, remote, client_key, client_cert, server_cert, |
135 | 138 |
@pass_context
|
136 | 139 |
def run_dummy(context):
|
137 | 140 |
"""
|
138 |
- Simple dummy client. Creates a session, accepts leases, does fake work and
|
|
139 |
- updates the server.
|
|
141 |
+ Creates a session, accepts leases, does fake work and updates the server.
|
|
140 | 142 |
"""
|
141 | 143 |
try:
|
142 |
- b = bot.Bot(context.bot_session)
|
|
144 |
+ b = bot.Bot(context.bot_session, context.update_period)
|
|
143 | 145 |
b.session(dummy.work_dummy,
|
144 | 146 |
context)
|
145 | 147 |
except KeyboardInterrupt:
|
146 | 148 |
pass
|
147 | 149 |
|
148 | 150 |
|
149 |
-@cli.command('temp-directory', short_help="Runs commands in temp directory and uploads results.")
|
|
151 |
+@cli.command('host-tools', short_help="Runs commands using the host's tools.")
|
|
150 | 152 |
@pass_context
|
151 |
-def run_temp_directory(context):
|
|
152 |
- """ Downloads files and command from CAS and runs
|
|
153 |
- in a temp directory, uploading result back to CAS
|
|
153 |
+def run_host_tools(context):
|
|
154 |
+ """
|
|
155 |
+ Downloads inputs from CAS, runs build commands using host-tools and uploads
|
|
156 |
+ result back to CAS.
|
|
154 | 157 |
"""
|
155 | 158 |
try:
|
156 |
- b = bot.Bot(context.bot_session)
|
|
157 |
- b.session(temp_directory.work_temp_directory,
|
|
159 |
+ b = bot.Bot(context.bot_session, context.update_period)
|
|
160 |
+ b.session(host.work_host_tools,
|
|
158 | 161 |
context)
|
159 | 162 |
except KeyboardInterrupt:
|
160 | 163 |
pass
|
... | ... | @@ -168,13 +171,13 @@ def run_temp_directory(context): |
168 | 171 |
@pass_context
|
169 | 172 |
def run_buildbox(context, local_cas, fuse_dir):
|
170 | 173 |
"""
|
171 |
- Uses BuildBox to run commands.
|
|
174 |
+ Uses BuildBox to run build commands.
|
|
172 | 175 |
"""
|
173 | 176 |
context.local_cas = local_cas
|
174 | 177 |
context.fuse_dir = fuse_dir
|
175 | 178 |
|
176 | 179 |
try:
|
177 |
- b = bot.Bot(context.bot_session)
|
|
180 |
+ b = bot.Bot(context.bot_session, context.update_period)
|
|
178 | 181 |
b.session(buildbox.work_buildbox,
|
179 | 182 |
context)
|
180 | 183 |
except KeyboardInterrupt:
|
... | ... | @@ -35,6 +35,9 @@ from ..cli import Context |
35 | 35 |
|
36 | 36 |
|
37 | 37 |
class YamlFactory(yaml.YAMLObject):
|
38 |
+ """ Base class for contructing maps or scalars from tags.
|
|
39 |
+ """
|
|
40 |
+ |
|
38 | 41 |
@classmethod
|
39 | 42 |
def from_yaml(cls, loader, node):
|
40 | 43 |
if isinstance(node, yaml.ScalarNode):
|
... | ... | @@ -47,6 +50,21 @@ class YamlFactory(yaml.YAMLObject): |
47 | 50 |
|
48 | 51 |
|
49 | 52 |
class Channel(YamlFactory):
|
53 |
+ """Creates a GRPC channel.
|
|
54 |
+ |
|
55 |
+ The :class:`Channel` class returns a `grpc.Channel` and is generated from the tag ``!channel``.
|
|
56 |
+ Creates either a secure or insecure channel.
|
|
57 |
+ |
|
58 |
+ Args:
|
|
59 |
+ port (int): A port for the channel.
|
|
60 |
+ insecure_mode (bool): If ``True``, generates an insecure channel, even if there are
|
|
61 |
+ credentials. Defaults to ``True``.
|
|
62 |
+ credentials (dict, optional): A dictionary in the form::
|
|
63 |
+ |
|
64 |
+ tls-server-key: /path/to/server-key
|
|
65 |
+ tls-server-cert: /path/to/server-cert
|
|
66 |
+ tls-client-certs: /path/to/client-certs
|
|
67 |
+ """
|
|
50 | 68 |
|
51 | 69 |
yaml_tag = u'!channel'
|
52 | 70 |
|
... | ... | @@ -69,6 +87,13 @@ class Channel(YamlFactory): |
69 | 87 |
|
70 | 88 |
|
71 | 89 |
class ExpandPath(YamlFactory):
|
90 |
+ """Returns a string of the user's path after expansion.
|
|
91 |
+ |
|
92 |
+ The :class:`ExpandPath` class returns a string and is generated from the tag ``!expand-path``.
|
|
93 |
+ |
|
94 |
+ Args:
|
|
95 |
+ path (str): Can be used with strings such as: ``~/dir/to/something`` or ``$HOME/certs``
|
|
96 |
+ """
|
|
72 | 97 |
|
73 | 98 |
yaml_tag = u'!expand-path'
|
74 | 99 |
|
... | ... | @@ -79,14 +104,30 @@ class ExpandPath(YamlFactory): |
79 | 104 |
|
80 | 105 |
|
81 | 106 |
class Disk(YamlFactory):
|
107 |
+ """Generates :class:`buildgrid.server.cas.storage.disk.DiskStorage` using the tag ``!disk-storage``.
|
|
108 |
+ |
|
109 |
+ Args:
|
|
110 |
+ path (str): Path to directory to storage.
|
|
111 |
+ |
|
112 |
+ """
|
|
82 | 113 |
|
83 | 114 |
yaml_tag = u'!disk-storage'
|
84 | 115 |
|
85 | 116 |
def __new__(cls, path):
|
117 |
+ """Creates a new disk
|
|
118 |
+ |
|
119 |
+ Args:
|
|
120 |
+ path (str): Some path
|
|
121 |
+ """
|
|
86 | 122 |
return DiskStorage(path)
|
87 | 123 |
|
88 | 124 |
|
89 | 125 |
class LRU(YamlFactory):
|
126 |
+ """Generates :class:`buildgrid.server.cas.storage.lru_memory_cache.LRUMemoryCache` using the tag ``!lru-storage``.
|
|
127 |
+ |
|
128 |
+ Args:
|
|
129 |
+ size (int): Size e.g ``10kb``. Size parsed with :meth:`buildgrid._app.settings.parser._parse_size`.
|
|
130 |
+ """
|
|
90 | 131 |
|
91 | 132 |
yaml_tag = u'!lru-storage'
|
92 | 133 |
|
... | ... | @@ -95,6 +136,12 @@ class LRU(YamlFactory): |
95 | 136 |
|
96 | 137 |
|
97 | 138 |
class S3(YamlFactory):
|
139 |
+ """Generates :class:`buildgrid.server.cas.storage.s3.S3Storage` using the tag ``!s3-storage``.
|
|
140 |
+ |
|
141 |
+ Args:
|
|
142 |
+ bucket (str): Name of bucket
|
|
143 |
+ endpoint (str): URL of endpoint.
|
|
144 |
+ """
|
|
98 | 145 |
|
99 | 146 |
yaml_tag = u'!s3-storage'
|
100 | 147 |
|
... | ... | @@ -103,6 +150,18 @@ class S3(YamlFactory): |
103 | 150 |
|
104 | 151 |
|
105 | 152 |
class Remote(YamlFactory):
|
153 |
+ """Generates :class:`buildgrid.server.cas.storage.remote.RemoteStorage`
|
|
154 |
+ using the tag ``!remote-storage``.
|
|
155 |
+ |
|
156 |
+ Args:
|
|
157 |
+ url (str): URL to remote storage. If used with ``https``, needs credentials.
|
|
158 |
+ instance_name (str): Instance of the remote to connect to.
|
|
159 |
+ credentials (dict, optional): A dictionary in the form::
|
|
160 |
+ |
|
161 |
+ tls-client-key: /path/to/client-key
|
|
162 |
+ tls-client-cert: /path/to/client-cert
|
|
163 |
+ tls-server-cert: /path/to/server-cert
|
|
164 |
+ """
|
|
106 | 165 |
|
107 | 166 |
yaml_tag = u'!remote-storage'
|
108 | 167 |
|
... | ... | @@ -144,6 +203,18 @@ class Remote(YamlFactory): |
144 | 203 |
|
145 | 204 |
|
146 | 205 |
class WithCache(YamlFactory):
|
206 |
+ """Generates :class:`buildgrid.server.cas.storage.with_cache.WithCacheStorage`
|
|
207 |
+ using the tag ``!with-cache-storage``.
|
|
208 |
+ |
|
209 |
+ Args:
|
|
210 |
+ url (str): URL to remote storage. If used with ``https``, needs credentials.
|
|
211 |
+ instance_name (str): Instance of the remote to connect to.
|
|
212 |
+ credentials (dict, optional): A dictionary in the form::
|
|
213 |
+ |
|
214 |
+ tls-client-key: /path/to/certs
|
|
215 |
+ tls-client-cert: /path/to/certs
|
|
216 |
+ tls-server-cert: /path/to/certs
|
|
217 |
+ """
|
|
147 | 218 |
|
148 | 219 |
yaml_tag = u'!with-cache-storage'
|
149 | 220 |
|
... | ... | @@ -152,6 +223,13 @@ class WithCache(YamlFactory): |
152 | 223 |
|
153 | 224 |
|
154 | 225 |
class Execution(YamlFactory):
|
226 |
+ """Generates :class:`buildgrid.server.execution.service.ExecutionService`
|
|
227 |
+ using the tag ``!execution``.
|
|
228 |
+ |
|
229 |
+ Args:
|
|
230 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
231 |
+ action_cache(:class:`Action`): Instance of action cache to use.
|
|
232 |
+ """
|
|
155 | 233 |
|
156 | 234 |
yaml_tag = u'!execution'
|
157 | 235 |
|
... | ... | @@ -160,6 +238,14 @@ class Execution(YamlFactory): |
160 | 238 |
|
161 | 239 |
|
162 | 240 |
class Action(YamlFactory):
|
241 |
+ """Generates :class:`buildgrid.server.actioncache.service.ActionCacheService`
|
|
242 |
+ using the tag ``!action-cache``.
|
|
243 |
+ |
|
244 |
+ Args:
|
|
245 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
246 |
+ max_cached_refs(int): Max number of cached actions.
|
|
247 |
+ allow_updates(bool): Allow updates pushed to CAS. Defaults to ``True``.
|
|
248 |
+ """
|
|
163 | 249 |
|
164 | 250 |
yaml_tag = u'!action-cache'
|
165 | 251 |
|
... | ... | @@ -168,6 +254,14 @@ class Action(YamlFactory): |
168 | 254 |
|
169 | 255 |
|
170 | 256 |
class Reference(YamlFactory):
|
257 |
+ """Generates :class:`buildgrid.server.referencestorage.service.ReferenceStorageService`
|
|
258 |
+ using the tag ``!reference-cache``.
|
|
259 |
+ |
|
260 |
+ Args:
|
|
261 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
262 |
+ max_cached_refs(int): Max number of cached actions.
|
|
263 |
+ allow_updates(bool): Allow updates pushed to CAS. Defauled to ``True``.
|
|
264 |
+ """
|
|
171 | 265 |
|
172 | 266 |
yaml_tag = u'!reference-cache'
|
173 | 267 |
|
... | ... | @@ -176,6 +270,12 @@ class Reference(YamlFactory): |
176 | 270 |
|
177 | 271 |
|
178 | 272 |
class CAS(YamlFactory):
|
273 |
+ """Generates :class:`buildgrid.server.cas.service.ContentAddressableStorageService`
|
|
274 |
+ using the tag ``!cas``.
|
|
275 |
+ |
|
276 |
+ Args:
|
|
277 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
278 |
+ """
|
|
179 | 279 |
|
180 | 280 |
yaml_tag = u'!cas'
|
181 | 281 |
|
... | ... | @@ -184,6 +284,12 @@ class CAS(YamlFactory): |
184 | 284 |
|
185 | 285 |
|
186 | 286 |
class ByteStream(YamlFactory):
|
287 |
+ """Generates :class:`buildgrid.server.cas.service.ByteStreamService`
|
|
288 |
+ using the tag ``!bytestream``.
|
|
289 |
+ |
|
290 |
+ Args:
|
|
291 |
+ storage(:class:`buildgrid.server.cas.storage.storage_abc.StorageABC`): Instance of storage to use.
|
|
292 |
+ """
|
|
187 | 293 |
|
188 | 294 |
yaml_tag = u'!bytestream'
|
189 | 295 |
|
... | ... | @@ -283,7 +283,7 @@ class Downloader: |
283 | 283 |
try:
|
284 | 284 |
batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
|
285 | 285 |
for response in batch_response.responses:
|
286 |
- assert response.digest.hash in digests
|
|
286 |
+ assert response.digest in digests
|
|
287 | 287 |
|
288 | 288 |
read_blobs.append(response.data)
|
289 | 289 |
|
... | ... | @@ -52,7 +52,7 @@ class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer): |
52 | 52 |
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
53 | 53 |
|
54 | 54 |
except NotFoundError as e:
|
55 |
- self.logger.error(e)
|
|
55 |
+ self.logger.info(e)
|
|
56 | 56 |
context.set_code(grpc.StatusCode.NOT_FOUND)
|
57 | 57 |
|
58 | 58 |
return remote_execution_pb2.ActionResult()
|
... | ... | @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk. |
21 | 21 |
"""
|
22 | 22 |
|
23 | 23 |
import os
|
24 |
-import pathlib
|
|
25 | 24 |
import tempfile
|
26 | 25 |
|
27 | 26 |
from .storage_abc import StorageABC
|
... | ... | @@ -30,28 +29,41 @@ from .storage_abc import StorageABC |
30 | 29 |
class DiskStorage(StorageABC):
|
31 | 30 |
|
32 | 31 |
def __init__(self, path):
|
33 |
- self._path = pathlib.Path(path)
|
|
34 |
- os.makedirs(str(self._path / "temp"), exist_ok=True)
|
|
32 |
+ if not os.path.isabs(path):
|
|
33 |
+ self.__root_path = os.path.abspath(path)
|
|
34 |
+ else:
|
|
35 |
+ self.__root_path = path
|
|
36 |
+ self.__cas_path = os.path.join(self.__root_path, 'cas')
|
|
37 |
+ |
|
38 |
+ self.objects_path = os.path.join(self.__cas_path, 'objects')
|
|
39 |
+ self.temp_path = os.path.join(self.__root_path, 'tmp')
|
|
40 |
+ |
|
41 |
+ os.makedirs(self.objects_path, exist_ok=True)
|
|
42 |
+ os.makedirs(self.temp_path, exist_ok=True)
|
|
35 | 43 |
|
36 | 44 |
def has_blob(self, digest):
|
37 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
|
|
45 |
+ return os.path.exists(self._get_object_path(digest))
|
|
38 | 46 |
|
39 | 47 |
def get_blob(self, digest):
|
40 | 48 |
try:
|
41 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
|
|
49 |
+ return open(self._get_object_path(digest), 'rb')
|
|
42 | 50 |
except FileNotFoundError:
|
43 | 51 |
return None
|
44 | 52 |
|
45 |
- def begin_write(self, _digest):
|
|
46 |
- return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
|
|
53 |
+ def begin_write(self, digest):
|
|
54 |
+ return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
|
|
47 | 55 |
|
48 | 56 |
def commit_write(self, digest, write_session):
|
49 |
- # Atomically move the temporary file into place.
|
|
50 |
- path = self._path / (digest.hash + "_" + str(digest.size_bytes))
|
|
51 |
- os.replace(write_session.name, str(path))
|
|
57 |
+ object_path = self._get_object_path(digest)
|
|
58 |
+ |
|
52 | 59 |
try:
|
53 |
- write_session.close()
|
|
54 |
- except FileNotFoundError:
|
|
55 |
- # We moved the temporary file to a new location, so when Python
|
|
56 |
- # tries to delete its old location, it'll fail.
|
|
60 |
+ os.makedirs(os.path.dirname(object_path), exist_ok=True)
|
|
61 |
+ os.link(write_session.name, object_path)
|
|
62 |
+ except FileExistsError:
|
|
63 |
+ # Object is already there!
|
|
57 | 64 |
pass
|
65 |
+ |
|
66 |
+ write_session.close()
|
|
67 |
+ |
|
68 |
+ def _get_object_path(self, digest):
|
|
69 |
+ return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])
|
... | ... | @@ -122,8 +122,11 @@ class Job: |
122 | 122 |
if self.result is not None:
|
123 | 123 |
self._operation.done = True
|
124 | 124 |
response = remote_execution_pb2.ExecuteResponse(result=self.result,
|
125 |
- cached_result=self.result_cached,
|
|
126 |
- status=self.lease.status)
|
|
125 |
+ cached_result=self.result_cached)
|
|
126 |
+ |
|
127 |
+ if not self.result_cached:
|
|
128 |
+ response.status.CopyFrom(self.lease.status)
|
|
129 |
+ |
|
127 | 130 |
self._operation.response.CopyFrom(self._pack_any(response))
|
128 | 131 |
|
129 | 132 |
return self._operation
|
... | ... | @@ -23,8 +23,6 @@ Schedules jobs. |
23 | 23 |
|
24 | 24 |
from collections import deque
|
25 | 25 |
|
26 |
-from google.protobuf import any_pb2
|
|
27 |
- |
|
28 | 26 |
from buildgrid._exceptions import NotFoundError
|
29 | 27 |
from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
|
30 | 28 |
from buildgrid._protos.google.longrunning import operations_pb2
|
... | ... | @@ -60,9 +58,7 @@ class Scheduler: |
60 | 58 |
job.update_execute_stage(ExecuteStage.QUEUED)
|
61 | 59 |
|
62 | 60 |
else:
|
63 |
- cached_result_any = any_pb2.Any()
|
|
64 |
- cached_result_any.Pack(cached_result)
|
|
65 |
- job.result = cached_result_any
|
|
61 |
+ job.result = cached_result
|
|
66 | 62 |
job.result_cached = True
|
67 | 63 |
job.update_execute_stage(ExecuteStage.COMPLETED)
|
68 | 64 |
|
1 |
+server:
|
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure_mode: true
|
|
5 |
+ |
|
6 |
+instances:
|
|
7 |
+ - name: main
|
|
8 |
+ |
|
9 |
+ storages:
|
|
10 |
+ - !disk-storage &main-storage
|
|
11 |
+ path: !expand-path $HOME/cas
|
|
12 |
+ |
|
13 |
+ services:
|
|
14 |
+ - !cas
|
|
15 |
+ storage: *main-storage
|
|
16 |
+ - !bytestream
|
|
17 |
+ storage: *main-storage
|
|
18 |
+ - !reference-cache
|
|
19 |
+ storage: *main-storage
|
|
20 |
+ max_cached_refs: 512
|
1 |
- |
|
2 | 1 |
.. _reference:
|
3 | 2 |
|
4 | 3 |
Reference
|
... | ... | @@ -11,3 +10,4 @@ This section contains BuildGrid API and CLI reference documentation. |
11 | 10 |
|
12 | 11 |
reference_api.rst
|
13 | 12 |
reference_cli.rst
|
13 |
+ reference_server_config.rst
|
... | ... | @@ -36,10 +36,10 @@ BuildGrid's Command Line Interface (CLI) reference documentation. |
36 | 36 |
|
37 | 37 |
----
|
38 | 38 |
|
39 |
-.. _invoking-bgd-bot-temp-directory:
|
|
39 |
+.. _invoking-bgd-bot-host-tools:
|
|
40 | 40 |
|
41 |
-.. click:: buildgrid._app.commands.cmd_bot:run_temp_directory
|
|
42 |
- :prog: bgd bot temp-directory
|
|
41 |
+.. click:: buildgrid._app.commands.cmd_bot:run_host_tools
|
|
42 |
+ :prog: bgd bot host-tools
|
|
43 | 43 |
|
44 | 44 |
----
|
45 | 45 |
|
... | ... | @@ -137,4 +137,4 @@ BuildGrid's Command Line Interface (CLI) reference documentation. |
137 | 137 |
.. _invoking-bgd-server-start:
|
138 | 138 |
|
139 | 139 |
.. click:: buildgrid._app.commands.cmd_server:start
|
140 |
- :prog: bgd server start
|
|
140 |
+ :prog: bgd server start
|
|
\ No newline at end of file |
1 |
+.. _parser:
|
|
2 |
+ |
|
3 |
+Server configuration reference
|
|
4 |
+==============================
|
|
5 |
+ |
|
6 |
+BuildGrid's server configuration. To be used with::
|
|
7 |
+ |
|
8 |
+ bgd server start server.conf
|
|
9 |
+ |
|
10 |
+.. automodule:: buildgrid._app.settings.parser
|
|
11 |
+ :members:
|
|
12 |
+ :undoc-members:
|
|
13 |
+ :show-inheritance:
|
1 |
- |
|
2 | 1 |
.. _using:
|
3 | 2 |
|
4 | 3 |
Using
|
... | ... | @@ -12,3 +11,5 @@ This section covers how to run an use the BuildGrid build service. |
12 | 11 |
using_internal.rst
|
13 | 12 |
using_bazel.rst
|
14 | 13 |
using_buildstream.rst
|
14 |
+ using_recc.rst
|
|
15 |
+ using_cas_server.rst
|
|
\ No newline at end of file |
... | ... | @@ -94,12 +94,12 @@ has ``gcc`` installed, run: |
94 | 94 |
|
95 | 95 |
.. code-block:: sh
|
96 | 96 |
|
97 |
- bgd bot --remote=http://localhost:50051 --parent=main temp-directory
|
|
97 |
+ bgd bot --remote=http://localhost:50051 --parent=main host-tools
|
|
98 | 98 |
|
99 | 99 |
The ``--remote`` option is used to specify the server location (running on the
|
100 | 100 |
same machine here, and listening to port 50051). The ``--parent`` option is used
|
101 |
-to specify the server instance you except the bot to be attached to. Refer to
|
|
102 |
-the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
|
|
101 |
+to specify the server instance you expect the bot to be attached to. Refer to
|
|
102 |
+the :ref:`CLI reference section <invoking-bgd-bot-host-tools>` for command
|
|
103 | 103 |
line interface details.
|
104 | 104 |
|
105 | 105 |
The BuildGrid server is now ready to accept jobs and execute them. Bazel needs
|
... | ... | @@ -120,7 +120,7 @@ You can finally have Bazel to build the example workspace by running: |
120 | 120 |
|
121 | 121 |
bazel build //main:hello-world
|
122 | 122 |
|
123 |
-You can verify that the example has been successfully build by running the
|
|
123 |
+You can verify that the example has been successfully built by running the
|
|
124 | 124 |
generated executable. Simply invoke:
|
125 | 125 |
|
126 | 126 |
.. code-block:: sh
|
... | ... | @@ -128,4 +128,4 @@ generated executable. Simply invoke: |
128 | 128 |
./bazel-bin/main/hello-world
|
129 | 129 |
|
130 | 130 |
.. _bazel-examples: https://github.com/bazelbuild/examples
|
131 |
-.. _stage3 CPP example: https://github.com/bazelbuild/examples/tree/master/cpp-tutorial/stage3
|
|
131 |
+.. _stage3 CPP example: https://github.com/bazelbuild/examples/tree/master/cpp-tutorial/stage3
|
|
\ No newline at end of file |
... | ... | @@ -149,7 +149,7 @@ that the ``buildbox`` tool is functional on your machine (refer to |
149 | 149 |
|
150 | 150 |
The ``--remote`` option is used to specify the server location (running on the
|
151 | 151 |
same machine here, and listening to port 50051). The ``--parent`` option is
|
152 |
-used to specify the server instance you except the bot to be attached to (empty
|
|
152 |
+used to specify the server instance you expect the bot to be attached to (empty
|
|
153 | 153 |
here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
|
154 | 154 |
and respectively specify the sandbox mount point and local CAS cache locations.
|
155 | 155 |
Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
|
... | ... | @@ -178,7 +178,7 @@ You can finally have BuildStream to build the example project by running: |
178 | 178 |
|
179 | 179 |
bst build hello.bst
|
180 | 180 |
|
181 |
-You can verify that the example has been successfully build by running the
|
|
181 |
+You can verify that the example has been successfully built by running the
|
|
182 | 182 |
generated executable. Simply invoke:
|
183 | 183 |
|
184 | 184 |
.. code-block:: sh
|
1 |
+.. _cas-server:
|
|
2 |
+ |
|
3 |
+CAS server
|
|
4 |
+==========
|
|
5 |
+ |
|
6 |
+It is possible to configure BuildGrid with just a Content Addressable Storage service.
|
|
7 |
+ |
|
8 |
+.. note::
|
|
9 |
+ |
|
10 |
+ This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
|
|
11 |
+ |
|
12 |
+.. _cas-configuration:
|
|
13 |
+ |
|
14 |
+Configuration
|
|
15 |
+-------------
|
|
16 |
+ |
|
17 |
+Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
|
|
18 |
+ |
|
19 |
+.. literalinclude:: ./data/cas-example-server.conf
|
|
20 |
+ :language: yaml
|
|
21 |
+ |
|
22 |
+.. hint::
|
|
23 |
+ |
|
24 |
+ Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
|
|
25 |
+ |
|
26 |
+This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
|
|
27 |
+ |
|
28 |
+.. code-block:: sh
|
|
29 |
+ |
|
30 |
+ bgd server start example.conf
|
|
31 |
+ |
|
32 |
+The server should now be available to use.
|
|
33 |
+ |
|
34 |
+.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
|
|
35 |
+.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto
|
1 |
+ |
|
1 | 2 |
.. _internal-client:
|
2 | 3 |
|
3 | 4 |
Internal client
|
... | ... | @@ -83,7 +84,7 @@ Start the following bot session: |
83 | 84 |
|
84 | 85 |
.. code-block:: sh
|
85 | 86 |
|
86 |
- bgd bot temp-directory
|
|
87 |
+ bgd bot host-tools
|
|
87 | 88 |
|
88 | 89 |
Upload the directory containing the C file:
|
89 | 90 |
|
... | ... | @@ -101,4 +102,4 @@ to run on the bot: |
101 | 102 |
bgd execute command --output-file hello True /path/to/test-buildgrid -- gcc -Wall hello.c -o hello
|
102 | 103 |
|
103 | 104 |
The resulting executeable should have returned to a new directory called
|
104 |
-``testing``.
|
|
105 |
+``testing``.
|
|
\ No newline at end of file |
1 |
+ |
|
2 |
+.. _recc-client:
|
|
3 |
+ |
|
4 |
+RECC client
|
|
5 |
+===========
|
|
6 |
+ |
|
7 |
+`RECC`_ is the *Remote Execution Caching Compiler*, an open source build tool
|
|
8 |
+that wraps compiler command calls and forwards them to a remote build execution
|
|
9 |
+service using the remote execution API (REAPI) v2.
|
|
10 |
+ |
|
11 |
+.. note::
|
|
12 |
+ |
|
13 |
+ There is no stable release of RECC yet. You'll have to `install it from
|
|
14 |
+ sources`_.
|
|
15 |
+ |
|
16 |
+.. _RECC: https://gitlab.com/bloomberg/recc
|
|
17 |
+.. _install it from sources: https://gitlab.com/bloomberg/recc#installing-dependencies
|
|
18 |
+ |
|
19 |
+ |
|
20 |
+.. _recc-configuration:
|
|
21 |
+ |
|
22 |
+Configuration
|
|
23 |
+-------------
|
|
24 |
+ |
|
25 |
+RECC reads the configuration from its execution environment. You can get a
|
|
26 |
+complete list of environment variables it accepts by running:
|
|
27 |
+ |
|
28 |
+.. code-block:: sh
|
|
29 |
+ |
|
30 |
+ recc --help
|
|
31 |
+ |
|
32 |
+The variables are prefixed with ``RECC_``. The most important ones for remote
|
|
33 |
+execution are:
|
|
34 |
+ |
|
35 |
+- ``RECC_SERVER``: URI of the remote execution server.
|
|
36 |
+- ``RECC_CAS_SERVER``: URI of the CAS server, defaults to ``RECC_SERVER``.
|
|
37 |
+- ``RECC_INSTANCE``: name of the remote execution instance.
|
|
38 |
+ |
|
39 |
+.. hint::
|
|
40 |
+ |
|
41 |
+ ``RECC_VERBOSE=1`` can be set in order to enable verbose output.
|
|
42 |
+ |
|
43 |
+As an example, in order to forward compile commands to the ``main`` instance of
|
|
44 |
+the remote execution server available at ``controller.grid.build`` on port
|
|
45 |
+``50051`` you should export:
|
|
46 |
+ |
|
47 |
+.. code-block:: sh
|
|
48 |
+ |
|
49 |
+ export RECC_SERVER=controller.grid.build:50051
|
|
50 |
+ export RECC_INSTANCE=main
|
|
51 |
+ |
|
52 |
+ |
|
53 |
+.. _recc-example:
|
|
54 |
+ |
|
55 |
+Example build
|
|
56 |
+-------------
|
|
57 |
+ |
|
58 |
+RECC can be use with any existing software package respecting `GNU make common
|
|
59 |
+variables`_ like ``CC`` for the C compiler or ``CXX`` for the C++ compiler.
|
|
60 |
+We'll focus here on instructions on how to build the `GNU Hello`_ example
|
|
61 |
+program using RECC and BuildGrid on your local machine.
|
|
62 |
+ |
|
63 |
+First, you need to download the hello source package:
|
|
64 |
+ |
|
65 |
+.. code-block:: sh
|
|
66 |
+ |
|
67 |
+ wget https://ftp.gnu.org/gnu/hello/hello-2.10.tar.gz
|
|
68 |
+ |
|
69 |
+Next, unpack it and change the current directory to the source root:
|
|
70 |
+ |
|
71 |
+.. code-block:: sh
|
|
72 |
+ |
|
73 |
+ tar xvf hello-2.10.tar.gz
|
|
74 |
+ cd hello-2.10
|
|
75 |
+ |
|
76 |
+.. hint::
|
|
77 |
+ |
|
78 |
+ All the commands in the instructions below are expected to be executed from
|
|
79 |
+ that root source directory (the GNU Hello project's root directory).
|
|
80 |
+ |
|
81 |
+Before trying to build the hello example program, you'll have to setup and run a
|
|
82 |
+BuildGrid server and bot. A minimal server's configuration is given below, paste
|
|
83 |
+it in a ``server.conf`` file in the root directory:
|
|
84 |
+ |
|
85 |
+.. literalinclude:: ./data/bazel-example-server.conf
|
|
86 |
+ :language: yaml
|
|
87 |
+ |
|
88 |
+This defines a single ``main`` server instance implementing a
|
|
89 |
+``ContentAddressableStorage`` (CAS) + ``ByteStream`` service together with an
|
|
90 |
+``Execution`` + ``ActionCache`` service, both using the same in-memory storage.
|
|
91 |
+You can then start the BuildGrid server daemon using that configuration by
|
|
92 |
+running:
|
|
93 |
+ |
|
94 |
+.. code-block:: sh
|
|
95 |
+ |
|
96 |
+ bgd server start server.conf
|
|
97 |
+ |
|
98 |
+In order to perform the actual build work, you need to attach a worker bot to
|
|
99 |
+that server for that ``main`` instance. RECC comes with its own ``reccworker``
|
|
100 |
+bot implementation. However, BuildGrid's host-tools based bot should be enough
|
|
101 |
+to build the hello example program. Once you've make sure that your machine has
|
|
102 |
+``gcc`` installed, run:
|
|
103 |
+ |
|
104 |
+.. code-block:: sh
|
|
105 |
+ |
|
106 |
+ bgd bot --remote=http://localhost:50051 --parent=main host-tools
|
|
107 |
+ |
|
108 |
+The ``--remote`` option is used to specify the server location (running on the
|
|
109 |
+same machine here, and listening to port 50051). The ``--parent`` option is used
|
|
110 |
+to specify the server instance you expect the bot to be attached to. Refer to
|
|
111 |
+the :ref:`CLI reference section <invoking-bgd-bot-host-tools>` for command
|
|
112 |
+line interface details.
|
|
113 |
+ |
|
114 |
+The BuildGrid server is now ready to accept jobs and execute them. RECC's
|
|
115 |
+:ref:`configuration <bazel-configuration>` needs to be defined as environment
|
|
116 |
+variables. Define minimal configuration by running:
|
|
117 |
+ |
|
118 |
+.. code-block:: sh
|
|
119 |
+ |
|
120 |
+ export RECC_SERVER=localhost:50051
|
|
121 |
+ export RECC_INSTANCE=main
|
|
122 |
+ |
|
123 |
+This points RECC to the ``main`` remote execution server instance at
|
|
124 |
+``localhost:50051``.
|
|
125 |
+ |
|
126 |
+GNU Hello is using `The Autotools`_ as a build system, so first, you need to
|
|
127 |
+configure your build. Run:
|
|
128 |
+ |
|
129 |
+.. code-block:: sh
|
|
130 |
+ |
|
131 |
+ ./configure
|
|
132 |
+ |
|
133 |
+You can finally build the hello example program, using RECC by running:
|
|
134 |
+ |
|
135 |
+.. code-block:: sh
|
|
136 |
+ |
|
137 |
+ make CC="recc cc"
|
|
138 |
+ |
|
139 |
+You can verify that the example program has been successfully built by running
|
|
140 |
+the generated executable. Simply invoke:
|
|
141 |
+ |
|
142 |
+.. code-block:: sh
|
|
143 |
+ |
|
144 |
+ ./hello
|
|
145 |
+ |
|
146 |
+.. _GNU make common variables: https://www.gnu.org/software/make/manual/html_node/Implicit-Variables.html
|
|
147 |
+.. _GNU Hello: https://www.gnu.org/software/hello
|
|
148 |
+.. _The Autotools: https://www.gnu.org/software/automake/manual/html_node/Autotools-Introduction.html
|
|
\ No newline at end of file |
... | ... | @@ -97,7 +97,8 @@ tests_require = [ |
97 | 97 |
]
|
98 | 98 |
|
99 | 99 |
docs_require = [
|
100 |
- 'sphinx',
|
|
100 |
+ # rtd-theme broken in Sphinx >= 1.8, this breaks search functionality.
|
|
101 |
+ 'sphinx == 1.7.8',
|
|
101 | 102 |
'sphinx-click',
|
102 | 103 |
'sphinx-rtd-theme',
|
103 | 104 |
'sphinxcontrib-apidoc',
|
... | ... | @@ -44,8 +44,8 @@ instances: |
44 | 44 |
The main server
|
45 | 45 |
|
46 | 46 |
storages:
|
47 |
- - !disk-storage &main-storage
|
|
48 |
- path: ~/cas/
|
|
47 |
+ - !lru-storage &main-storage
|
|
48 |
+ size: 256mb
|
|
49 | 49 |
|
50 | 50 |
services:
|
51 | 51 |
- !action-cache &main-action
|