finn pushed to branch finn/remove-disk-storage-from-tests at BuildGrid / buildgrid
Commits:
-
febe2ced
by Martin Blanchard at 2018-10-03T08:08:56Z
-
e80b6503
by Martin Blanchard at 2018-10-03T08:08:56Z
-
a305c9a1
by Finn at 2018-10-03T09:22:30Z
-
18c50d1f
by Finn at 2018-10-03T09:30:11Z
4 changed files:
- buildgrid/client/cas.py
- buildgrid/server/actioncache/service.py
- buildgrid/server/cas/storage/disk.py
- tests/server_instance.py
Changes:
| ... | ... | @@ -283,7 +283,7 @@ class Downloader: |
| 283 | 283 |
try:
|
| 284 | 284 |
batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
|
| 285 | 285 |
for response in batch_response.responses:
|
| 286 |
- assert response.digest.hash in digests
|
|
| 286 |
+ assert response.digest in digests
|
|
| 287 | 287 |
|
| 288 | 288 |
read_blobs.append(response.data)
|
| 289 | 289 |
|
| ... | ... | @@ -52,7 +52,7 @@ class ActionCacheService(remote_execution_pb2_grpc.ActionCacheServicer): |
| 52 | 52 |
context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
|
| 53 | 53 |
|
| 54 | 54 |
except NotFoundError as e:
|
| 55 |
- self.logger.error(e)
|
|
| 55 |
+ self.logger.info(e)
|
|
| 56 | 56 |
context.set_code(grpc.StatusCode.NOT_FOUND)
|
| 57 | 57 |
|
| 58 | 58 |
return remote_execution_pb2.ActionResult()
|
| ... | ... | @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk. |
| 21 | 21 |
"""
|
| 22 | 22 |
|
| 23 | 23 |
import os
|
| 24 |
-import pathlib
|
|
| 25 | 24 |
import tempfile
|
| 26 | 25 |
|
| 27 | 26 |
from .storage_abc import StorageABC
|
| ... | ... | @@ -30,28 +29,41 @@ from .storage_abc import StorageABC |
| 30 | 29 |
class DiskStorage(StorageABC):
|
| 31 | 30 |
|
| 32 | 31 |
def __init__(self, path):
|
| 33 |
- self._path = pathlib.Path(path)
|
|
| 34 |
- os.makedirs(str(self._path / "temp"), exist_ok=True)
|
|
| 32 |
+ if not os.path.isabs(path):
|
|
| 33 |
+ self.__root_path = os.path.abspath(path)
|
|
| 34 |
+ else:
|
|
| 35 |
+ self.__root_path = path
|
|
| 36 |
+ self.__cas_path = os.path.join(self.__root_path, 'cas')
|
|
| 37 |
+ |
|
| 38 |
+ self.objects_path = os.path.join(self.__cas_path, 'objects')
|
|
| 39 |
+ self.temp_path = os.path.join(self.__root_path, 'tmp')
|
|
| 40 |
+ |
|
| 41 |
+ os.makedirs(self.objects_path, exist_ok=True)
|
|
| 42 |
+ os.makedirs(self.temp_path, exist_ok=True)
|
|
| 35 | 43 |
|
| 36 | 44 |
def has_blob(self, digest):
|
| 37 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
|
|
| 45 |
+ return os.path.exists(self._get_object_path(digest))
|
|
| 38 | 46 |
|
| 39 | 47 |
def get_blob(self, digest):
|
| 40 | 48 |
try:
|
| 41 |
- return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
|
|
| 49 |
+ return open(self._get_object_path(digest), 'rb')
|
|
| 42 | 50 |
except FileNotFoundError:
|
| 43 | 51 |
return None
|
| 44 | 52 |
|
| 45 |
- def begin_write(self, _digest):
|
|
| 46 |
- return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
|
|
| 53 |
+ def begin_write(self, digest):
|
|
| 54 |
+ return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
|
|
| 47 | 55 |
|
| 48 | 56 |
def commit_write(self, digest, write_session):
|
| 49 |
- # Atomically move the temporary file into place.
|
|
| 50 |
- path = self._path / (digest.hash + "_" + str(digest.size_bytes))
|
|
| 51 |
- os.replace(write_session.name, str(path))
|
|
| 57 |
+ object_path = self._get_object_path(digest)
|
|
| 58 |
+ |
|
| 52 | 59 |
try:
|
| 53 |
- write_session.close()
|
|
| 54 |
- except FileNotFoundError:
|
|
| 55 |
- # We moved the temporary file to a new location, so when Python
|
|
| 56 |
- # tries to delete its old location, it'll fail.
|
|
| 60 |
+ os.makedirs(os.path.dirname(object_path), exist_ok=True)
|
|
| 61 |
+ os.link(write_session.name, object_path)
|
|
| 62 |
+ except FileExistsError:
|
|
| 63 |
+ # Object is already there!
|
|
| 57 | 64 |
pass
|
| 65 |
+ |
|
| 66 |
+ write_session.close()
|
|
| 67 |
+ |
|
| 68 |
+ def _get_object_path(self, digest):
|
|
| 69 |
+ return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])
|
| ... | ... | @@ -44,8 +44,8 @@ instances: |
| 44 | 44 |
The main server
|
| 45 | 45 |
|
| 46 | 46 |
storages:
|
| 47 |
- - !disk-storage &main-storage
|
|
| 48 |
- path: ~/cas/
|
|
| 47 |
+ - !lru-storage &main-storage
|
|
| 48 |
+ size: 256mb
|
|
| 49 | 49 |
|
| 50 | 50 |
services:
|
| 51 | 51 |
- !action-cache &main-action
|