GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/cas-docs] 5 commits: Locked sphinx to version 1.7.8.

Title: GitLab

finn pushed to branch finn/cas-docs at BuildGrid / buildgrid

Commits:

8 changed files:

Changes:

  • buildgrid/client/cas.py
    ... ... @@ -283,7 +283,7 @@ class Downloader:
    283 283 
                 try:
    284 284 
                     batch_response = self.__cas_stub.BatchReadBlobs(batch_request)
    285 285 
                     for response in batch_response.responses:
    286 
    -                    assert response.digest.hash in digests
    286 
    +                    assert response.digest in digests
    287 287
    288 288 
                         read_blobs.append(response.data)
    289 289

  • buildgrid/server/cas/storage/disk.py
    ... ... @@ -21,7 +21,6 @@ A CAS storage provider that stores files as blobs on disk.
    21 21 
     """
    22 22
    23 23 
     import os
    24 
    -import pathlib
    25 24 
     import tempfile
    26 25
    27 26 
     from .storage_abc import StorageABC
    ... ... @@ -30,28 +29,41 @@ from .storage_abc import StorageABC
    30 29 
     class DiskStorage(StorageABC):
    31 30
    32 31 
         def __init__(self, path):
    33 
    -        self._path = pathlib.Path(path)
    34 
    -        os.makedirs(str(self._path / "temp"), exist_ok=True)
    32 
    +        if not os.path.isabs(path):
    33 
    +            self.__root_path = os.path.abspath(path)
    34 
    +        else:
    35 
    +            self.__root_path = path
    36 
    +        self.__cas_path = os.path.join(self.__root_path, 'cas')
    37 
    +
    38 
    +        self.objects_path = os.path.join(self.__cas_path, 'objects')
    39 
    +        self.temp_path = os.path.join(self.__root_path, 'tmp')
    40 
    +
    41 
    +        os.makedirs(self.objects_path, exist_ok=True)
    42 
    +        os.makedirs(self.temp_path, exist_ok=True)
    35 43
    36 44 
         def has_blob(self, digest):
    37 
    -        return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
    45 
    +        return os.path.exists(self._get_object_path(digest))
    38 46
    39 47 
         def get_blob(self, digest):
    40 48 
             try:
    41 
    -            return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
    49 
    +            return open(self._get_object_path(digest), 'rb')
    42 50 
             except FileNotFoundError:
    43 51 
                 return None
    44 52
    45 
    -    def begin_write(self, _digest):
    46 
    -        return tempfile.NamedTemporaryFile("wb", dir=str(self._path / "temp"))
    53 
    +    def begin_write(self, digest):
    54 
    +        return tempfile.NamedTemporaryFile("wb", dir=self.temp_path)
    47 55
    48 56 
         def commit_write(self, digest, write_session):
    49 
    -        # Atomically move the temporary file into place.
    50 
    -        path = self._path / (digest.hash + "_" + str(digest.size_bytes))
    51 
    -        os.replace(write_session.name, str(path))
    57 
    +        object_path = self._get_object_path(digest)
    58 
    +
    52 59 
             try:
    53 
    -            write_session.close()
    54 
    -        except FileNotFoundError:
    55 
    -            # We moved the temporary file to a new location, so when Python
    56 
    -            # tries to delete its old location, it'll fail.
    60 
    +            os.makedirs(os.path.dirname(object_path), exist_ok=True)
    61 
    +            os.link(write_session.name, object_path)
    62 
    +        except FileExistsError:
    63 
    +            # Object is already there!
    57 64 
                 pass
    65 
    +
    66 
    +        write_session.close()
    67 
    +
    68 
    +    def _get_object_path(self, digest):
    69 
    +        return os.path.join(self.objects_path, digest.hash[:2], digest.hash[2:])

  • docs/source/data/cas-example-server.conf
    1 
    +server:
    2 
    +  - !channel
    3 
    +    port: 50051
    4 
    +    insecure_mode: true
    5 
    +
    6 
    +instances:
    7 
    +  - name: main
    8 
    +
    9 
    +    storages:
    10 
    +      - !disk-storage &main-storage
    11 
    +	path: !expand-path $HOME/cas
    12 
    +
    13 
    +    services:
    14 
    +      - !cas
    15 
    +	storage: *main-storage
    16 
    +      - !bytestream
    17 
    +	storage: *main-storage
    18 
    +      - !reference-cache
    19 
    +	storage: *main-storage
    20 
    +	max_cached_refs: 512

  • docs/source/using.rst
    1 
    -
    2 1 
     .. _using:
    3 2
    4 3 
     Using
    ... ... @@ -12,3 +11,4 @@ This section covers how to run an use the BuildGrid build service.
    12 11 
        using_internal.rst
    13 12 
        using_bazel.rst
    14 13 
        using_buildstream.rst
    14 
    +   using_cas_server.rst

  • docs/source/using_bazel.rst
    ... ... @@ -98,7 +98,7 @@ has ``gcc`` installed, run:
    98 98
    99 99 
     The ``--remote`` option is used to specify the server location (running on the
    100 100 
     same machine here, and listening to port 50051). The ``--parent`` option is used
    101 
    -to specify the server instance you except the bot to be attached to. Refer to
    101 
    +to specify the server instance you expect the bot to be attached to. Refer to
    102 102 
     the :ref:`CLI reference section <invoking-bgd-bot-temp-directory>` for command
    103 103 
     line interface details.
    104 104
    ... ... @@ -120,7 +120,7 @@ You can finally have Bazel to build the example workspace by running:
    120 120
    121 121 
        bazel build //main:hello-world
    122 122
    123 
    -You can verify that the example has been successfully build by running the
    123 
    +You can verify that the example has been successfully built by running the
    124 124 
     generated executable. Simply invoke:
    125 125
    126 126 
     .. code-block:: sh

  • docs/source/using_buildstream.rst
    ... ... @@ -149,7 +149,7 @@ that the ``buildbox`` tool is functional on your machine (refer to
    149 149
    150 150 
     The ``--remote`` option is used to specify the server location (running on the
    151 151 
     same machine here, and listening to port 50051). The ``--parent`` option is
    152 
    -used to specify the server instance you except the bot to be attached to (empty
    152 
    +used to specify the server instance you expect the bot to be attached to (empty
    153 153 
     here). ``--fuse-dir`` and ``--local-cas`` are specific to the ``buildbox`` bot
    154 154 
     and respectively specify the sandbox mount point and local CAS cache locations.
    155 155 
     Refer to the :ref:`CLI reference section <invoking-bgd-bot-buildbox>` for
    ... ... @@ -178,7 +178,7 @@ You can finally have BuildStream to build the example project by running:
    178 178
    179 179 
        bst build hello.bst
    180 180
    181 
    -You can verify that the example has been successfully build by running the
    181 
    +You can verify that the example has been successfully built by running the
    182 182 
     generated executable. Simply invoke:
    183 183
    184 184 
     .. code-block:: sh

  • docs/source/using_cas_server.rst
    1 
    +.. _cas-server:
    2 
    +
    3 
    +CAS server
    4 
    +==========
    5 
    +
    6 
    +It is possible to configure BuildGrid with just a Content Addressable Storage service.
    7 
    +
    8 
    +.. note::
    9 
    +
    10 
    +   This service can be equivalent to `BuildStream's Artifact Server`_ if the `Reference Storage Service`_ is included.
    11 
    +
    12 
    +.. _cas-configuration:
    13 
    +
    14 
    +Configuration
    15 
    +-------------
    16 
    +
    17 
    +Here is an example project configuration. It also implements an optional API called the `Reference Storage Service`_, which if used, allows the user to store a ``Digest`` behind a user defined ``key``.
    18 
    +
    19 
    +.. literalinclude:: ./data/cas-example-server.conf
    20 
    +   :language: yaml
    21 
    +
    22 
    +.. hint::
    23 
    +
    24 
    +   Use ``- name: ""`` if using with BuildStream, as instance names are not supported for that tool yet.
    25 
    +
    26 
    +This defines a single ``main`` instance of the ``CAS``, ``Bytestream`` and ``Reference Storage`` service on port ``55051``. It is backed onto disk storage and will populate the folder ``$HOME/cas``. To start the server, simply type into your terminal:
    27 
    +
    28 
    +.. code-block:: sh
    29 
    +
    30 
    +   bgd server start example.conf
    31 
    +
    32 
    +The server should now be available to use.
    33 
    +
    34 
    +.. _BuildStream's Artifact Server: https://buildstream.gitlab.io/buildstream/install_artifacts.html
    35 
    +.. _Reference Storage Service: https://gitlab.com/BuildGrid/buildgrid/blob/master/buildgrid/_protos/buildstream/v2/buildstream.proto

  • setup.py
    ... ... @@ -97,7 +97,8 @@ tests_require = [
    97 97 
     ]
    98 98
    99 99 
     docs_require = [
    100 
    -    'sphinx',
    100 
    +    # rtd-theme broken in Sphinx >= 1.8, this breaks search functionality.
    101 
    +    'sphinx == 1.7.8',
    101 102 
         'sphinx-click',
    102 103 
         'sphinx-rtd-theme',
    103 104 
         'sphinxcontrib-apidoc',

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]