richardmaw-codethink pushed to branch richardmaw/builddir-sockets at BuildStream / buildstream
Commits:
-
47f3064a
by Jürg Billeter at 2018-09-10T15:43:25Z
-
1a7fb3cb
by Jürg Billeter at 2018-09-10T15:43:25Z
-
b3ffcdc8
by Jürg Billeter at 2018-09-10T16:07:30Z
-
6f925bcb
by Javier Jardón at 2018-09-13T07:38:48Z
-
c6155f8d
by Javier Jardón at 2018-09-13T08:11:54Z
-
19838a07
by Chandan Singh at 2018-09-13T10:58:38Z
-
3b81d451
by Chandan Singh at 2018-09-13T12:27:59Z
-
55956762
by Richard Maw at 2018-09-13T16:50:33Z
-
fc7f83ac
by richardmaw-codethink at 2018-09-13T17:14:31Z
-
233a7d83
by Richard Maw at 2018-09-14T08:53:14Z
-
f06f234a
by Richard Maw at 2018-09-14T08:53:14Z
9 changed files:
- README.rst
- buildstream/_artifactcache/cascache.py
- buildstream/_artifactcache/casserver.py
- buildstream/element.py
- buildstream/utils.py
- doc/source/install_source.rst
- + tests/integration/project/elements/sockets/make-builddir-socket.bst
- + tests/integration/project/elements/sockets/make-install-root-socket.bst
- + tests/integration/sockets.py
Changes:
... | ... | @@ -13,6 +13,9 @@ About |
13 | 13 |
.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
|
14 | 14 |
:target: https://gitlab.com/BuildStream/buildstream/commits/master
|
15 | 15 |
|
16 |
+.. image:: https://img.shields.io/pypi/v/BuildStream.svg
|
|
17 |
+ :target: https://pypi.org/project/BuildStream
|
|
18 |
+ |
|
16 | 19 |
|
17 | 20 |
What is BuildStream?
|
18 | 21 |
====================
|
... | ... | @@ -684,6 +684,9 @@ class CASCache(ArtifactCache): |
684 | 684 |
symlinknode = directory.symlinks.add()
|
685 | 685 |
symlinknode.name = name
|
686 | 686 |
symlinknode.target = os.readlink(full_path)
|
687 |
+ elif stat.S_ISSOCK(mode):
|
|
688 |
+ # The process serving the socket can't be cached anyway
|
|
689 |
+ pass
|
|
687 | 690 |
else:
|
688 | 691 |
raise ArtifactError("Unsupported file type for {}".format(full_path))
|
689 | 692 |
|
... | ... | @@ -38,6 +38,10 @@ from .._context import Context |
38 | 38 |
from .cascache import CASCache
|
39 | 39 |
|
40 | 40 |
|
41 |
+# The default limit for gRPC messages is 4 MiB
|
|
42 |
+_MAX_BATCH_TOTAL_SIZE_BYTES = 4 * 1024 * 1024
|
|
43 |
+ |
|
44 |
+ |
|
41 | 45 |
# Trying to push an artifact that is too large
|
42 | 46 |
class ArtifactTooLargeException(Exception):
|
43 | 47 |
pass
|
... | ... | @@ -67,6 +71,9 @@ def create_server(repo, *, enable_push): |
67 | 71 |
remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
|
68 | 72 |
_ContentAddressableStorageServicer(artifactcache), server)
|
69 | 73 |
|
74 |
+ remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
|
|
75 |
+ _CapabilitiesServicer(), server)
|
|
76 |
+ |
|
70 | 77 |
buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
|
71 | 78 |
_ReferenceStorageServicer(artifactcache, enable_push=enable_push), server)
|
72 | 79 |
|
... | ... | @@ -229,6 +236,48 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres |
229 | 236 |
d.size_bytes = digest.size_bytes
|
230 | 237 |
return response
|
231 | 238 |
|
239 |
+ def BatchReadBlobs(self, request, context):
|
|
240 |
+ response = remote_execution_pb2.BatchReadBlobsResponse()
|
|
241 |
+ batch_size = 0
|
|
242 |
+ |
|
243 |
+ for digest in request.digests:
|
|
244 |
+ batch_size += digest.size_bytes
|
|
245 |
+ if batch_size > _MAX_BATCH_TOTAL_SIZE_BYTES:
|
|
246 |
+ context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
|
247 |
+ return response
|
|
248 |
+ |
|
249 |
+ blob_response = response.responses.add()
|
|
250 |
+ blob_response.digest.hash = digest.hash
|
|
251 |
+ blob_response.digest.size_bytes = digest.size_bytes
|
|
252 |
+ try:
|
|
253 |
+ with open(self.cas.objpath(digest), 'rb') as f:
|
|
254 |
+ if os.fstat(f.fileno()).st_size != digest.size_bytes:
|
|
255 |
+ blob_response.status.code = grpc.StatusCode.NOT_FOUND
|
|
256 |
+ continue
|
|
257 |
+ |
|
258 |
+ blob_response.data = f.read(digest.size_bytes)
|
|
259 |
+ except FileNotFoundError:
|
|
260 |
+ blob_response.status.code = grpc.StatusCode.NOT_FOUND
|
|
261 |
+ |
|
262 |
+ return response
|
|
263 |
+ |
|
264 |
+ |
|
265 |
+class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
|
|
266 |
+ def GetCapabilities(self, request, context):
|
|
267 |
+ response = remote_execution_pb2.ServerCapabilities()
|
|
268 |
+ |
|
269 |
+ cache_capabilities = response.cache_capabilities
|
|
270 |
+ cache_capabilities.digest_function.append(remote_execution_pb2.SHA256)
|
|
271 |
+ cache_capabilities.action_cache_update_capabilities.update_enabled = False
|
|
272 |
+ cache_capabilities.max_batch_total_size_bytes = _MAX_BATCH_TOTAL_SIZE_BYTES
|
|
273 |
+ cache_capabilities.symlink_absolute_path_strategy = remote_execution_pb2.CacheCapabilities.ALLOWED
|
|
274 |
+ |
|
275 |
+ response.deprecated_api_version.major = 2
|
|
276 |
+ response.low_api_version.major = 2
|
|
277 |
+ response.high_api_version.major = 2
|
|
278 |
+ |
|
279 |
+ return response
|
|
280 |
+ |
|
232 | 281 |
|
233 | 282 |
class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
|
234 | 283 |
def __init__(self, cas, *, enable_push):
|
... | ... | @@ -200,7 +200,6 @@ class Element(Plugin): |
200 | 200 |
self.__strict_cache_key = None # Our cached cache key for strict builds
|
201 | 201 |
self.__artifacts = artifacts # Artifact cache
|
202 | 202 |
self.__consistency = Consistency.INCONSISTENT # Cached overall consistency state
|
203 |
- self.__cached = None # Whether we have a cached artifact
|
|
204 | 203 |
self.__strong_cached = None # Whether we have a cached artifact
|
205 | 204 |
self.__weak_cached = None # Whether we have a cached artifact
|
206 | 205 |
self.__assemble_scheduled = False # Element is scheduled to be assembled
|
... | ... | @@ -1126,8 +1125,6 @@ class Element(Plugin): |
1126 | 1125 |
|
1127 | 1126 |
# Query caches now that the weak and strict cache keys are available
|
1128 | 1127 |
key_for_cache_lookup = self.__strict_cache_key if context.get_strict() else self.__weak_cache_key
|
1129 |
- if not self.__cached:
|
|
1130 |
- self.__cached = self.__artifacts.contains(self, key_for_cache_lookup)
|
|
1131 | 1128 |
if not self.__strong_cached:
|
1132 | 1129 |
self.__strong_cached = self.__artifacts.contains(self, self.__strict_cache_key)
|
1133 | 1130 |
if key_for_cache_lookup == self.__weak_cache_key:
|
... | ... | @@ -2079,7 +2076,7 @@ class Element(Plugin): |
2079 | 2076 |
|
2080 | 2077 |
def __is_cached(self, keystrength):
|
2081 | 2078 |
if keystrength is None:
|
2082 |
- return self.__cached
|
|
2079 |
+ keystrength = _KeyStrength.STRONG if self._get_context().get_strict() else _KeyStrength.WEAK
|
|
2083 | 2080 |
|
2084 | 2081 |
return self.__strong_cached if keystrength == _KeyStrength.STRONG else self.__weak_cached
|
2085 | 2082 |
|
... | ... | @@ -372,6 +372,8 @@ def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa |
372 | 372 |
Directories in `dest` are replaced with files from `src`,
|
373 | 373 |
unless the existing directory in `dest` is not empty in which
|
374 | 374 |
case the path will be reported in the return value.
|
375 |
+ |
|
376 |
+ UNIX domain socket files from `src` are ignored.
|
|
375 | 377 |
"""
|
376 | 378 |
presorted = False
|
377 | 379 |
if files is None:
|
... | ... | @@ -414,6 +416,8 @@ def link_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa |
414 | 416 |
|
415 | 417 |
If a hardlink cannot be created due to crossing filesystems,
|
416 | 418 |
then the file will be copied instead.
|
419 |
+ |
|
420 |
+ UNIX domain socket files from `src` are ignored.
|
|
417 | 421 |
"""
|
418 | 422 |
presorted = False
|
419 | 423 |
if files is None:
|
... | ... | @@ -841,6 +845,13 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result, |
841 | 845 |
os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
|
842 | 846 |
os.chmod(destpath, file_stat.st_mode)
|
843 | 847 |
|
848 |
+ elif stat.S_ISFIFO(mode):
|
|
849 |
+ os.mkfifo(destpath, mode)
|
|
850 |
+ |
|
851 |
+ elif stat.S_ISSOCK(mode):
|
|
852 |
+ # We can't duplicate the process serving the socket anyway
|
|
853 |
+ pass
|
|
854 |
+ |
|
844 | 855 |
else:
|
845 | 856 |
# Unsupported type.
|
846 | 857 |
raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath))
|
... | ... | @@ -29,6 +29,7 @@ The default plugins with extra host dependencies are: |
29 | 29 |
* git
|
30 | 30 |
* ostree
|
31 | 31 |
* patch
|
32 |
+* pip
|
|
32 | 33 |
* tar
|
33 | 34 |
|
34 | 35 |
If you intend to push built artifacts to a remote artifact server,
|
1 |
+kind: manual
|
|
2 |
+ |
|
3 |
+depends:
|
|
4 |
+- filename: base.bst
|
|
5 |
+ type: build
|
|
6 |
+ |
|
7 |
+config:
|
|
8 |
+ build-commands:
|
|
9 |
+ - |
|
|
10 |
+ python3 -c '
|
|
11 |
+ from socket import socket, AF_UNIX, SOCK_STREAM
|
|
12 |
+ s = socket(AF_UNIX, SOCK_STREAM)
|
|
13 |
+ s.bind("testsocket")
|
|
14 |
+ '
|
1 |
+kind: manual
|
|
2 |
+ |
|
3 |
+depends:
|
|
4 |
+- filename: base.bst
|
|
5 |
+ type: build
|
|
6 |
+ |
|
7 |
+config:
|
|
8 |
+ install-commands:
|
|
9 |
+ - |
|
|
10 |
+ python3 -c '
|
|
11 |
+ from os.path import join
|
|
12 |
+ from sys import argv
|
|
13 |
+ from socket import socket, AF_UNIX, SOCK_STREAM
|
|
14 |
+ s = socket(AF_UNIX, SOCK_STREAM)
|
|
15 |
+ s.bind(join(argv[1], "testsocket"))
|
|
16 |
+ ' %{install-root}
|
1 |
+import os
|
|
2 |
+import pytest
|
|
3 |
+ |
|
4 |
+from buildstream import _yaml
|
|
5 |
+ |
|
6 |
+from tests.testutils import cli_integration as cli
|
|
7 |
+from tests.testutils.integration import assert_contains
|
|
8 |
+ |
|
9 |
+ |
|
10 |
+pytestmark = pytest.mark.integration
|
|
11 |
+ |
|
12 |
+DATA_DIR = os.path.join(
|
|
13 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
14 |
+ "project"
|
|
15 |
+)
|
|
16 |
+ |
|
17 |
+ |
|
18 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
19 |
+def test_builddir_socket_ignored(cli, tmpdir, datafiles):
|
|
20 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
21 |
+ element_name = 'sockets/make-builddir-socket.bst'
|
|
22 |
+ |
|
23 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
24 |
+ assert result.exit_code == 0
|
|
25 |
+ |
|
26 |
+ |
|
27 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
28 |
+def test_install_root_socket_ignored(cli, tmpdir, datafiles):
|
|
29 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
30 |
+ element_name = 'sockets/make-install-root-socket.bst'
|
|
31 |
+ |
|
32 |
+ result = cli.run(project=project, args=['build', element_name])
|
|
33 |
+ assert result.exit_code == 0
|