finn pushed to branch finn/cas-docs at BuildGrid / buildgrid
Commits:
-
9401c87d
by Finn at 2018-10-02T16:33:03Z
-
51f661fd
by Jürg Billeter at 2018-10-03T07:50:34Z
-
febe2ced
by Martin Blanchard at 2018-10-03T08:08:56Z
-
e80b6503
by Martin Blanchard at 2018-10-03T08:08:56Z
-
62dffc49
by Finn at 2018-10-03T09:23:31Z
8 changed files:
- buildgrid/client/cas.py
- buildgrid/server/cas/storage/disk.py
- + docs/source/data/cas-example-server.conf
- docs/source/using.rst
- docs/source/using_bazel.rst
- docs/source/using_buildstream.rst
- + docs/source/using_cas_server.rst
- setup.py
Changes:
... | ... | @@ -283,7 +283,7 @@ class Downloader: |
283 | 283 |
try:
|
284 | 284 |
batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
|
285 | 285 |
for response in batch_response.responses:
|
286 |
- assert response.digest.hash in digests
|
|
286 |
+ assert response.digest in digests
|
|
287 | 287 |
|
288 | 288 |
read_blobs.append(response.data)
|
289 | 289 |
|
... | ... | @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk. |
21 | 21 |
"""
|
22 | 22 |
|
23 | 23 |
import os
|
24 |
-import pathlib
|
|
25 | 24 |
import tempfile
|
26 | 25 |
|
27 | 26 |
from .storage_abc import StorageABC
|
... | ... | @@ -30,28 +29,41 @@ from .storage_abc import StorageABC |
30 | 29 |
class DiskStorage(StorageABC):
|
31 | 30 |
|
32 | 31 |
def __init__(self, path):
|
33 |
- self._path = pathlib.Path(path)
|
|
34 |
- os.makedirs(str(self._path / "temp"), exist_ok=True)
|
|
32 |
+ if not os.path.isabs(path):
|
|
33 |
+ self.__root_path = os.path.abspath(path)
|
|
34 |
+ else:
|
|
35 |
+ self.__root_path = path
|
|
36 |
+ self.__cas_path = os.path.join(self.__root_path, 'cas')
|
|
37 |
+ |
|
38 |
+ self.objects_path = os.path.join(self.__cas_path, 'objects')
|
|
39 |
+ self.temp_path = os.path.join(self.__root_path, 'tmp')
|
|
40 |
+ |
|
41 |
+ os.makedirs(self.objects_path, exist_ok=True)
|
|
42 |
+ os.makedirs(self.temp_path, exist_ok=True)
|
|
35 | 43 |
|
36 | 44 |
def has_blob(self, digest):
|
37 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
|
|
45 |
+ return os.path.exists(self._get_object_path(digest))
|
|
38 | 46 |
|
39 | 47 |
def get_blob(self, digest):
|
40 | 48 |
try:
|
41 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
|
|
49 |
+ return open(self._get_object_path(digest), 'rb')
|
|
42 | 50 |
except FileNotFoundError:
|
43 | 51 |
return None
|
44 | 52 |
|
45 |
- def begin_write(self, _digest):
|
|
46 |
- return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
|
|
53 |
+ def begin_write(self, digest):
|
|
54 |
+ return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
|
|
47 | 55 |
|
48 | 56 |
def commit_write(self, digest, write_session):
|
49 |
- # Atomically move the temporary file into place.
|
|
50 |
- path = self._path / (digest.hash + "_" + str(digest.size_bytes))
|
|
51 |
- os.replace(write_session.name, str(path))
|
|
57 |
+ object_path = self._get_object_path(digest)
|
|
58 |
+ |
|
52 | 59 |
try:
|
53 |
- write_session.close()
|
|
54 |
- except FileNotFoundError:
|
|
55 |
- # We moved the temporary file to a new location, so when Python
|
|
56 |
- # tries to delete its old location, it'll fail.
|
|
60 |
+ os.makedirs(os.path.dirname(object_path), exist_ok=True)
|
|
61 |
+ os.link(write_session.name, object_path)
|
|
62 |
+ except FileExistsError:
|
|
63 |
+ # Object is already there!
|
|
57 | 64 |
pass
|
65 |
+ |
|
66 |
+ write_session.close()
|
|
67 |
+ |
|
68 |
+ def _get_object_path(self, digest):
|
|
69 |
+ return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])
|
1 |
+server:
|
|
2 |
+ - !channel
|
|
3 |
+ port: 50051
|
|
4 |
+ insecure_mode: true
|
|
5 |
+ |
|
6 |
+instances:
|
|
7 |
+ - name: main
|
|
8 |
+ |
|
9 |
+ storages:
|
|
10 |
+ - !disk-storage &main-storage
|
|
11 |
+ path: !expand-path $HOME/cas
|
|
12 |
+ |
|
13 |
+ services:
|
|
14 |
+ - !cas
|
|
15 |
+ storage: *main-storage
|
|
16 |
+ - !bytestream
|
|
17 |
+ storage: *main-storage
|
|
18 |
+ - !reference-cache
|
|
19 |
+ storage: *main-storage
|
|
20 |
+ max_cached_refs: 512
|
1 |
- |
|
2 | 1 |
.. _using:
|
3 | 2 |
|
4 | 3 |
Using
|
... | ... | @@ -12,3 +11,4 @@ This section covers how to run an use the BuildGrid build service. |
12 | 11 |
using_internal.rst
|
13 | 12 |
using_bazel.rst
|
14 | 13 |
using_buildstream.rst
|
14 |
+ using_cas_server.rst
|
... | ... | @@ -98,7 +98,7 @@ has ``gcc`` installed, run: |
98 | 98 |
|
99 | 99 |
The ``--remote`` option is used to specify the server location (running on the
|
100 | 100 |
same machine here, and listening to port 50051). The ``--parent`` option is used
|
101 |
-to specify the server instance you except the bot to be attached to. Refer to
|
|
101 |
+to specify the server instance you expect the bot to be attached to. Refer to
|
|
102 | 102 |
the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
|
103 | 103 |
line interface details.
|
104 | 104 |
|
... | ... | @@ -120,7 +120,7 @@ You can finally have Bazel to build the example workspace by running: |
120 | 120 |
|
121 | 121 |
bazel build //main:hello-world
|
122 | 122 |
|
123 |
-You can verify that the example has been successfully build by running the
|
|
123 |
+You can verify that the example has been successfully built by running the
|
|
124 | 124 |
generated executable. Simply invoke:
|
125 | 125 |
|
126 | 126 |
.. code-block:: sh
|
... | ... | @@ -149,7 +149,7 @@ that the ``buildbox`` tool is functional on your machine (refer to |
149 | 149 |
|
150 | 150 |
The ``--remote`` option is used to specify the server location (running on the
|
151 | 151 |
same machine here, and listening to port 50051). The ``--parent`` option is
|
152 |
-used to specify the server instance you except the bot to be attached to (empty
|
|
152 |
+used to specify the server instance you expect the bot to be attached to (empty
|
|
153 | 153 |
here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
|
154 | 154 |
and respectively specify the sandbox mount point and local CAS cache locations.
|
155 | 155 |
Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
|
... | ... | @@ -178,7 +178,7 @@ You can finally have BuildStream to build the example project by running: |
178 | 178 |
|
179 | 179 |
bst build hello.bst
|
180 | 180 |
|
181 |
-You can verify that the example has been successfully build by running the
|
|
181 |
+You can verify that the example has been successfully built by running the
|
|
182 | 182 |
generated executable. Simply invoke:
|
183 | 183 |
|
184 | 184 |
.. code-block:: sh
|
1 |
+.. _cas-server:
|
|
2 |
+ |
|
3 |
+CAS server
|
|
4 |
+==========
|
|
5 |
+ |
|
6 |
+It is possible to configure BuildGrid with just a Content Addressable Storage service.
|
|
7 |
+ |
|
8 |
+.. note::
|
|
9 |
+ |
|
10 |
+ This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
|
|
11 |
+ |
|
12 |
+.. _cas-configuration:
|
|
13 |
+ |
|
14 |
+Configuration
|
|
15 |
+-------------
|
|
16 |
+ |
|
17 |
+Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
|
|
18 |
+ |
|
19 |
+.. literalinclude:: ./data/cas-example-server.conf
|
|
20 |
+ :language: yaml
|
|
21 |
+ |
|
22 |
+.. hint::
|
|
23 |
+ |
|
24 |
+ Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
|
|
25 |
+ |
|
26 |
+This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
|
|
27 |
+ |
|
28 |
+.. code-block:: sh
|
|
29 |
+ |
|
30 |
+ bgd server start example.conf
|
|
31 |
+ |
|
32 |
+The server should now be available to use.
|
|
33 |
+ |
|
34 |
+.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
|
|
35 |
+.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto
|
... | ... | @@ -97,7 +97,8 @@ tests_require = [ |
97 | 97 |
]
|
98 | 98 |
|
99 | 99 |
docs_require = [
|
100 |
- 'sphinx',
|
|
100 |
+ # rtd-theme broken in Sphinx >= 1.8, this breaks search functionality.
|
|
101 |
+ 'sphinx == 1.7.8',
|
|
101 | 102 |
'sphinx-click',
|
102 | 103 |
'sphinx-rtd-theme',
|
103 | 104 |
'sphinxcontrib-apidoc',
|