[Notes] [Git][BuildGrid/buildgrid][master] 2 commits: storage/disk.py: Port to BuildStream compatible path scheme



Title: GitLab

Martin Blanchard pushed to branch master at BuildGrid / buildgrid

Commits:

2 changed files:

Changes:

  • buildgrid/client/cas.py
    ... ... @@ -283,7 +283,7 @@ class Downloader:
    283 283
                 try:
    
    284 284
                     batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
    
    285 285
                     for response in batch_response.responses:
    
    286
    -                    assert response.digest.hash in digests
    
    286
    +                    assert response.digest in digests
    
    287 287
     
    
    288 288
                         read_blobs.append(response.data)
    
    289 289
     
    

  • buildgrid/server/cas/storage/disk.py
    ... ... @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk.
    21 21
     """
    
    22 22
     
    
    23 23
     import os
    
    24
    -import pathlib
    
    25 24
     import tempfile
    
    26 25
     
    
    27 26
     from .storage_abc import StorageABC
    
    ... ... @@ -30,28 +29,41 @@ from .storage_abc import StorageABC
    30 29
     class DiskStorage(StorageABC):
    
    31 30
     
    
    32 31
         def __init__(self, path):
    
    33
    -        self._path = pathlib.Path(path)
    
    34
    -        os.makedirs(str(self._path / "temp"), exist_ok=True)
    
    32
    +        if not os.path.isabs(path):
    
    33
    +            self.__root_path = os.path.abspath(path)
    
    34
    +        else:
    
    35
    +            self.__root_path = path
    
    36
    +        self.__cas_path = os.path.join(self.__root_path, 'cas')
    
    37
    +
    
    38
    +        self.objects_path = os.path.join(self.__cas_path, 'objects')
    
    39
    +        self.temp_path = os.path.join(self.__root_path, 'tmp')
    
    40
    +
    
    41
    +        os.makedirs(self.objects_path, exist_ok=True)
    
    42
    +        os.makedirs(self.temp_path, exist_ok=True)
    
    35 43
     
    
    36 44
         def has_blob(self, digest):
    
    37
    -        return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
    
    45
    +        return os.path.exists(self._get_object_path(digest))
    
    38 46
     
    
    39 47
         def get_blob(self, digest):
    
    40 48
             try:
    
    41
    -            return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
    
    49
    +            return open(self._get_object_path(digest), 'rb')
    
    42 50
             except FileNotFoundError:
    
    43 51
                 return None
    
    44 52
     
    
    45
    -    def begin_write(self, _digest):
    
    46
    -        return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
    
    53
    +    def begin_write(self, digest):
    
    54
    +        return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
    
    47 55
     
    
    48 56
         def commit_write(self, digest, write_session):
    
    49
    -        # Atomically move the temporary file into place.
    
    50
    -        path = self._path / (digest.hash + "_" + str(digest.size_bytes))
    
    51
    -        os.replace(write_session.name, str(path))
    
    57
    +        object_path = self._get_object_path(digest)
    
    58
    +
    
    52 59
             try:
    
    53
    -            write_session.close()
    
    54
    -        except FileNotFoundError:
    
    55
    -            # We moved the temporary file to a new location, so when Python
    
    56
    -            # tries to delete its old location, it'll fail.
    
    60
    +            os.makedirs(os.path.dirname(object_path), exist_ok=True)
    
    61
    +            os.link(write_session.name, object_path)
    
    62
    +        except FileExistsError:
    
    63
    +            # Object is already there!
    
    57 64
                 pass
    
    65
    +
    
    66
    +        write_session.close()
    
    67
    +
    
    68
    +    def _get_object_path(self, digest):
    
    69
    +        return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]