[Notes] [Git][BuildStream/buildstream][juerg/platform] 23 commits: ci: add ACME challenge



Title: GitLab

Jürg Billeter pushed to branch juerg/platform at BuildStream / buildstream

Commits:

21 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -273,8 +273,14 @@ pages:
    273 273
       dependencies:
    
    274 274
       - source_dist
    
    275 275
       - docs
    
    276
    +  variables:
    
    277
    +    ACME_DIR: public/.well-known/acme-challenge
    
    276 278
       script:
    
    277
    -  - find public/
    
    279
    +  - mkdir -p ${ACME_DIR}
    
    280
    +    # Required to finish the creation of the Let's Encrypt certificate,
    
    281
    +    # which allows using https://docs.buildstream.build/ for accessing
    
    282
    +    # the documentation.
    
    283
    +  - echo ${ACME_CHALLENGE} > ${ACME_DIR}/$(echo ${ACME_CHALLENGE} | cut -c1-43)
    
    278 284
       artifacts:
    
    279 285
         paths:
    
    280 286
         - public/
    

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -383,6 +383,13 @@ class ArtifactCache():
    383 383
         # Abstract methods for subclasses to implement #
    
    384 384
         ################################################
    
    385 385
     
    
    386
    +    # preflight():
    
    387
    +    #
    
    388
    +    # Preflight check.
    
    389
    +    #
    
    390
    +    def preflight(self):
    
    391
    +        pass
    
    392
    +
    
    386 393
         # update_atime()
    
    387 394
         #
    
    388 395
         # Update the atime of an artifact.
    

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -54,7 +54,6 @@ _MAX_PAYLOAD_BYTES = 1024 * 1024
    54 54
     #
    
    55 55
     # Args:
    
    56 56
     #     context (Context): The BuildStream context
    
    57
    -#     enable_push (bool): Whether pushing is allowed by the platform
    
    58 57
     #
    
    59 58
     # Pushing is explicitly disabled by the platform in some cases,
    
    60 59
     # like when we are falling back to functioning without using
    
    ... ... @@ -62,7 +61,7 @@ _MAX_PAYLOAD_BYTES = 1024 * 1024
    62 61
     #
    
    63 62
     class CASCache(ArtifactCache):
    
    64 63
     
    
    65
    -    def __init__(self, context, *, enable_push=True):
    
    64
    +    def __init__(self, context):
    
    66 65
             super().__init__(context)
    
    67 66
     
    
    68 67
             self.casdir = os.path.join(context.artifactdir, 'cas')
    
    ... ... @@ -71,8 +70,6 @@ class CASCache(ArtifactCache):
    71 70
     
    
    72 71
             self._calculate_cache_quota()
    
    73 72
     
    
    74
    -        self._enable_push = enable_push
    
    75
    -
    
    76 73
             # Per-project list of _CASRemote instances.
    
    77 74
             self._remotes = {}
    
    78 75
     
    
    ... ... @@ -83,6 +80,12 @@ class CASCache(ArtifactCache):
    83 80
         #     Implementation of abstract methods       #
    
    84 81
         ################################################
    
    85 82
     
    
    83
    +    def preflight(self):
    
    84
    +        if (not os.path.isdir(os.path.join(self.casdir, 'refs', 'heads')) or
    
    85
    +            not os.path.isdir(os.path.join(self.casdir, 'objects'))):
    
    86
    +            raise ArtifactError("CAS repository check failed for '{}'"
    
    87
    +                                .format(self.casdir))
    
    88
    +
    
    86 89
         def contains(self, element, key):
    
    87 90
             refpath = self._refpath(self.get_artifact_fullname(element, key))
    
    88 91
     
    
    ... ... @@ -214,7 +217,7 @@ class CASCache(ArtifactCache):
    214 217
                 return bool(remotes_for_project)
    
    215 218
     
    
    216 219
         def has_push_remotes(self, *, element=None):
    
    217
    -        if not self._has_push_remotes or not self._enable_push:
    
    220
    +        if not self._has_push_remotes:
    
    218 221
                 # No project has push remotes
    
    219 222
                 return False
    
    220 223
             elif element is None:
    

  • buildstream/_artifactcache/casserver.py
    ... ... @@ -35,8 +35,6 @@ from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
    35 35
     from .._exceptions import ArtifactError
    
    36 36
     from .._context import Context
    
    37 37
     
    
    38
    -from .cascache import CASCache
    
    39
    -
    
    40 38
     
    
    41 39
     # The default limit for gRPC messages is 4 MiB.
    
    42 40
     # Limit payload to 1 MiB to leave sufficient headroom for metadata.
    
    ... ... @@ -60,7 +58,7 @@ def create_server(repo, *, enable_push):
    60 58
         context = Context()
    
    61 59
         context.artifactdir = os.path.abspath(repo)
    
    62 60
     
    
    63
    -    artifactcache = CASCache(context)
    
    61
    +    artifactcache = context.artifactcache
    
    64 62
     
    
    65 63
         # Use max_workers default from Python 3.5+
    
    66 64
         max_workers = (os.cpu_count() or 1) * 5
    

  • buildstream/_context.py
    ... ... @@ -30,6 +30,7 @@ from ._exceptions import LoadError, LoadErrorReason, BstError
    30 30
     from ._message import Message, MessageType
    
    31 31
     from ._profile import Topics, profile_start, profile_end
    
    32 32
     from ._artifactcache import ArtifactCache
    
    33
    +from ._artifactcache.cascache import CASCache
    
    33 34
     from ._workspaces import Workspaces
    
    34 35
     from .plugin import _plugin_lookup
    
    35 36
     
    
    ... ... @@ -113,6 +114,7 @@ class Context():
    113 114
             self._cache_key = None
    
    114 115
             self._message_handler = None
    
    115 116
             self._message_depth = deque()
    
    117
    +        self._artifactcache = None
    
    116 118
             self._projects = []
    
    117 119
             self._project_overrides = {}
    
    118 120
             self._workspaces = None
    
    ... ... @@ -227,6 +229,13 @@ class Context():
    227 229
                                 "{}: on-error should be one of: {}".format(
    
    228 230
                                     provenance, ", ".join(valid_actions)))
    
    229 231
     
    
    232
    +    @property
    
    233
    +    def artifactcache(self):
    
    234
    +        if not self._artifactcache:
    
    235
    +            self._artifactcache = CASCache(self)
    
    236
    +
    
    237
    +        return self._artifactcache
    
    238
    +
    
    230 239
         # add_project():
    
    231 240
         #
    
    232 241
         # Add a project to the context.
    

  • buildstream/_frontend/app.py
    ... ... @@ -198,10 +198,15 @@ class App():
    198 198
                 if option_value is not None:
    
    199 199
                     setattr(self.context, context_attr, option_value)
    
    200 200
             try:
    
    201
    -            Platform.create_instance(self.context)
    
    201
    +            Platform.get_platform()
    
    202 202
             except BstError as e:
    
    203 203
                 self._error_exit(e, "Error instantiating platform")
    
    204 204
     
    
    205
    +        try:
    
    206
    +            self.context.artifactcache.preflight()
    
    207
    +        except BstError as e:
    
    208
    +            self._error_exit(e, "Error instantiating artifact cache")
    
    209
    +
    
    205 210
             # Create the logger right before setting the message handler
    
    206 211
             self.logger = LogLine(self.context,
    
    207 212
                                   self._content_profile,
    

  • buildstream/_loader/loader.py
    ... ... @@ -28,7 +28,6 @@ from .. import Consistency
    28 28
     from .. import _yaml
    
    29 29
     from ..element import Element
    
    30 30
     from .._profile import Topics, profile_start, profile_end
    
    31
    -from .._platform import Platform
    
    32 31
     from .._includes import Includes
    
    33 32
     
    
    34 33
     from .types import Symbol, Dependency
    
    ... ... @@ -518,8 +517,7 @@ class Loader():
    518 517
                 raise LoadError(LoadErrorReason.INVALID_DATA,
    
    519 518
                                 "{}: Expected junction but element kind is {}".format(filename, meta_element.kind))
    
    520 519
     
    
    521
    -        platform = Platform.get_platform()
    
    522
    -        element = Element._new_from_meta(meta_element, platform.artifactcache)
    
    520
    +        element = Element._new_from_meta(meta_element, self._context.artifactcache)
    
    523 521
             element._preflight()
    
    524 522
     
    
    525 523
             sources = list(element.sources())
    

  • buildstream/_platform/linux.py
    ... ... @@ -17,11 +17,11 @@
    17 17
     #  Authors:
    
    18 18
     #        Tristan Maat <tristan maat codethink co uk>
    
    19 19
     
    
    20
    +import os
    
    20 21
     import subprocess
    
    21 22
     
    
    22 23
     from .. import _site
    
    23 24
     from .. import utils
    
    24
    -from .._artifactcache.cascache import CASCache
    
    25 25
     from .._message import Message, MessageType
    
    26 26
     from ..sandbox import SandboxBwrap
    
    27 27
     
    
    ... ... @@ -30,17 +30,15 @@ from . import Platform
    30 30
     
    
    31 31
     class Linux(Platform):
    
    32 32
     
    
    33
    -    def __init__(self, context):
    
    33
    +    def __init__(self):
    
    34 34
     
    
    35
    -        super().__init__(context)
    
    35
    +        super().__init__()
    
    36 36
     
    
    37
    -        self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    
    38
    -        self._user_ns_available = self._check_user_ns_available(context)
    
    39
    -        self._artifact_cache = CASCache(context, enable_push=self._user_ns_available)
    
    37
    +        self._uid = os.geteuid()
    
    38
    +        self._gid = os.getegid()
    
    40 39
     
    
    41
    -    @property
    
    42
    -    def artifactcache(self):
    
    43
    -        return self._artifact_cache
    
    40
    +        self._die_with_parent_available = _site.check_bwrap_version(0, 1, 8)
    
    41
    +        self._user_ns_available = self._check_user_ns_available()
    
    44 42
     
    
    45 43
         def create_sandbox(self, *args, **kwargs):
    
    46 44
             # Inform the bubblewrap sandbox as to whether it can use user namespaces or not
    
    ... ... @@ -48,10 +46,19 @@ class Linux(Platform):
    48 46
             kwargs['die_with_parent_available'] = self._die_with_parent_available
    
    49 47
             return SandboxBwrap(*args, **kwargs)
    
    50 48
     
    
    49
    +    def check_sandbox_config(self, config):
    
    50
    +        if self._user_ns_available:
    
    51
    +            # User namespace support allows arbitrary build UID/GID settings.
    
    52
    +            return True
    
    53
    +        else:
    
    54
    +            # Without user namespace support, the UID/GID in the sandbox
    
    55
    +            # will match the host UID/GID.
    
    56
    +            return config.build_uid == self._uid and config.build_gid == self._gid
    
    57
    +
    
    51 58
         ################################################
    
    52 59
         #              Private Methods                 #
    
    53 60
         ################################################
    
    54
    -    def _check_user_ns_available(self, context):
    
    61
    +    def _check_user_ns_available(self):
    
    55 62
     
    
    56 63
             # Here, lets check if bwrap is able to create user namespaces,
    
    57 64
             # issue a warning if it's not available, and save the state
    
    ... ... @@ -75,9 +82,4 @@ class Linux(Platform):
    75 82
                 return True
    
    76 83
     
    
    77 84
             else:
    
    78
    -            context.message(
    
    79
    -                Message(None, MessageType.WARN,
    
    80
    -                        "Unable to create user namespaces with bubblewrap, resorting to fallback",
    
    81
    -                        detail="Some builds may not function due to lack of uid / gid 0, " +
    
    82
    -                        "artifacts created will not be trusted for push purposes."))
    
    83 85
                 return False

  • buildstream/_platform/platform.py
    ... ... @@ -29,17 +29,13 @@ class Platform():
    29 29
         # Platform()
    
    30 30
         #
    
    31 31
         # A class to manage platform-specific details. Currently holds the
    
    32
    -    # sandbox factory, the artifact cache and staging operations, as
    
    33
    -    # well as platform helpers.
    
    32
    +    # sandbox factory as well as platform helpers.
    
    34 33
         #
    
    35
    -    # Args:
    
    36
    -    #     context (context): The project context
    
    37
    -    #
    
    38
    -    def __init__(self, context):
    
    39
    -        self.context = context
    
    34
    +    def __init__(self):
    
    35
    +        pass
    
    40 36
     
    
    41 37
         @classmethod
    
    42
    -    def create_instance(cls, *args, **kwargs):
    
    38
    +    def _create_instance(cls):
    
    43 39
             if sys.platform.startswith('linux'):
    
    44 40
                 backend = 'linux'
    
    45 41
             else:
    
    ... ... @@ -58,22 +54,14 @@ class Platform():
    58 54
             else:
    
    59 55
                 raise PlatformError("No such platform: '{}'".format(backend))
    
    60 56
     
    
    61
    -        cls._instance = PlatformImpl(*args, **kwargs)
    
    57
    +        cls._instance = PlatformImpl()
    
    62 58
     
    
    63 59
         @classmethod
    
    64 60
         def get_platform(cls):
    
    65 61
             if not cls._instance:
    
    66
    -            raise PlatformError("Platform needs to be initialized first")
    
    62
    +            cls._create_instance()
    
    67 63
             return cls._instance
    
    68 64
     
    
    69
    -    ##################################################################
    
    70
    -    #                       Platform properties                      #
    
    71
    -    ##################################################################
    
    72
    -    @property
    
    73
    -    def artifactcache(self):
    
    74
    -        raise ImplError("Platform {platform} does not implement an artifactcache"
    
    75
    -                        .format(platform=type(self).__name__))
    
    76
    -
    
    77 65
         ##################################################################
    
    78 66
         #                        Sandbox functions                       #
    
    79 67
         ##################################################################
    
    ... ... @@ -92,3 +80,7 @@ class Platform():
    92 80
         def create_sandbox(self, *args, **kwargs):
    
    93 81
             raise ImplError("Platform {platform} does not implement create_sandbox()"
    
    94 82
                             .format(platform=type(self).__name__))
    
    83
    +
    
    84
    +    def check_sandbox_config(self, config):
    
    85
    +        raise ImplError("Platform {platform} does not implement check_sandbox_config()"
    
    86
    +                        .format(platform=type(self).__name__))

  • buildstream/_platform/unix.py
    ... ... @@ -19,7 +19,6 @@
    19 19
     
    
    20 20
     import os
    
    21 21
     
    
    22
    -from .._artifactcache.cascache import CASCache
    
    23 22
     from .._exceptions import PlatformError
    
    24 23
     from ..sandbox import SandboxChroot
    
    25 24
     
    
    ... ... @@ -28,18 +27,21 @@ from . import Platform
    28 27
     
    
    29 28
     class Unix(Platform):
    
    30 29
     
    
    31
    -    def __init__(self, context):
    
    30
    +    def __init__(self):
    
    32 31
     
    
    33
    -        super().__init__(context)
    
    34
    -        self._artifact_cache = CASCache(context)
    
    32
    +        super().__init__()
    
    33
    +
    
    34
    +        self._uid = os.geteuid()
    
    35
    +        self._gid = os.getegid()
    
    35 36
     
    
    36 37
             # Not necessarily 100% reliable, but we want to fail early.
    
    37
    -        if os.geteuid() != 0:
    
    38
    +        if self._uid != 0:
    
    38 39
                 raise PlatformError("Root privileges are required to run without bubblewrap.")
    
    39 40
     
    
    40
    -    @property
    
    41
    -    def artifactcache(self):
    
    42
    -        return self._artifact_cache
    
    43
    -
    
    44 41
         def create_sandbox(self, *args, **kwargs):
    
    45 42
             return SandboxChroot(*args, **kwargs)
    
    43
    +
    
    44
    +    def check_sandbox_config(self, config):
    
    45
    +        # With the chroot sandbox, the UID/GID in the sandbox
    
    46
    +        # will match the host UID/GID (typically 0/0).
    
    47
    +        return config.build_uid == self._uid and config.build_gid == self._gid

  • buildstream/_scheduler/jobs/cachesizejob.py
    ... ... @@ -17,7 +17,6 @@
    17 17
     #        Tristan Daniël Maat <tristan maat codethink co uk>
    
    18 18
     #
    
    19 19
     from .job import Job
    
    20
    -from ..._platform import Platform
    
    21 20
     
    
    22 21
     
    
    23 22
     class CacheSizeJob(Job):
    
    ... ... @@ -25,8 +24,8 @@ class CacheSizeJob(Job):
    25 24
             super().__init__(*args, **kwargs)
    
    26 25
             self._complete_cb = complete_cb
    
    27 26
     
    
    28
    -        platform = Platform.get_platform()
    
    29
    -        self._artifacts = platform.artifactcache
    
    27
    +        context = self._scheduler.context
    
    28
    +        self._artifacts = context.artifactcache
    
    30 29
     
    
    31 30
         def child_process(self):
    
    32 31
             return self._artifacts.compute_cache_size()
    

  • buildstream/_scheduler/jobs/cleanupjob.py
    ... ... @@ -17,15 +17,14 @@
    17 17
     #        Tristan Daniël Maat <tristan maat codethink co uk>
    
    18 18
     #
    
    19 19
     from .job import Job
    
    20
    -from ..._platform import Platform
    
    21 20
     
    
    22 21
     
    
    23 22
     class CleanupJob(Job):
    
    24 23
         def __init__(self, *args, **kwargs):
    
    25 24
             super().__init__(*args, **kwargs)
    
    26 25
     
    
    27
    -        platform = Platform.get_platform()
    
    28
    -        self._artifacts = platform.artifactcache
    
    26
    +        context = self._scheduler.context
    
    27
    +        self._artifacts = context.artifactcache
    
    29 28
     
    
    30 29
         def child_process(self):
    
    31 30
             return self._artifacts.clean()
    

  • buildstream/_scheduler/queues/buildqueue.py
    ... ... @@ -24,7 +24,6 @@ from . import Queue, QueueStatus
    24 24
     from ..jobs import ElementJob
    
    25 25
     from ..resources import ResourceType
    
    26 26
     from ..._message import MessageType
    
    27
    -from ..._platform import Platform
    
    28 27
     
    
    29 28
     
    
    30 29
     # A queue which assembles elements
    
    ... ... @@ -94,8 +93,8 @@ class BuildQueue(Queue):
    94 93
             # as returned from Element._assemble() to the estimated
    
    95 94
             # artifact cache size
    
    96 95
             #
    
    97
    -        platform = Platform.get_platform()
    
    98
    -        artifacts = platform.artifactcache
    
    96
    +        context = self._scheduler.context
    
    97
    +        artifacts = context.artifactcache
    
    99 98
     
    
    100 99
             artifacts.add_artifact_size(artifact_size)
    
    101 100
     
    

  • buildstream/_scheduler/scheduler.py
    ... ... @@ -29,7 +29,6 @@ from contextlib import contextmanager
    29 29
     # Local imports
    
    30 30
     from .resources import Resources, ResourceType
    
    31 31
     from .jobs import CacheSizeJob, CleanupJob
    
    32
    -from .._platform import Platform
    
    33 32
     
    
    34 33
     
    
    35 34
     # A decent return code for Scheduler.run()
    
    ... ... @@ -348,8 +347,8 @@ class Scheduler():
    348 347
         #       which will report the calculated cache size.
    
    349 348
         #
    
    350 349
         def _run_cleanup(self, cache_size):
    
    351
    -        platform = Platform.get_platform()
    
    352
    -        artifacts = platform.artifactcache
    
    350
    +        context = self.context
    
    351
    +        artifacts = context.artifactcache
    
    353 352
     
    
    354 353
             if not artifacts.has_quota_exceeded():
    
    355 354
                 return
    

  • buildstream/_stream.py
    ... ... @@ -32,7 +32,6 @@ from ._exceptions import StreamError, ImplError, BstError, set_last_task_error
    32 32
     from ._message import Message, MessageType
    
    33 33
     from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, BuildQueue, PullQueue, PushQueue
    
    34 34
     from ._pipeline import Pipeline, PipelineSelection
    
    35
    -from ._platform import Platform
    
    36 35
     from . import utils, _yaml, _site
    
    37 36
     from . import Scope, Consistency
    
    38 37
     
    
    ... ... @@ -71,8 +70,7 @@ class Stream():
    71 70
             #
    
    72 71
             # Private members
    
    73 72
             #
    
    74
    -        self._platform = Platform.get_platform()
    
    75
    -        self._artifacts = self._platform.artifactcache
    
    73
    +        self._artifacts = context.artifactcache
    
    76 74
             self._context = context
    
    77 75
             self._project = project
    
    78 76
             self._pipeline = Pipeline(context, project, self._artifacts)
    

  • buildstream/element.py
    ... ... @@ -246,15 +246,23 @@ class Element(Plugin):
    246 246
             self.__config = self.__extract_config(meta)
    
    247 247
             self._configure(self.__config)
    
    248 248
     
    
    249
    -        # Extract Sandbox config
    
    250
    -        self.__sandbox_config = self.__extract_sandbox_config(meta)
    
    251
    -
    
    252 249
             # Extract remote execution URL
    
    253 250
             if not self.__is_junction:
    
    254 251
                 self.__remote_execution_url = project.remote_execution_url
    
    255 252
             else:
    
    256 253
                 self.__remote_execution_url = None
    
    257 254
     
    
    255
    +        # Extract Sandbox config
    
    256
    +        self.__sandbox_config = self.__extract_sandbox_config(meta)
    
    257
    +
    
    258
    +        self.__sandbox_config_supported = True
    
    259
    +        if not self.__use_remote_execution():
    
    260
    +            platform = Platform.get_platform()
    
    261
    +            if not platform.check_sandbox_config(self.__sandbox_config):
    
    262
    +                # Local sandbox does not fully support specified sandbox config.
    
    263
    +                # This will taint the artifact, disable pushing.
    
    264
    +                self.__sandbox_config_supported = False
    
    265
    +
    
    258 266
         def __lt__(self, other):
    
    259 267
             return self.name < other.name
    
    260 268
     
    
    ... ... @@ -1521,6 +1529,11 @@ class Element(Plugin):
    1521 1529
             context = self._get_context()
    
    1522 1530
             with self._output_file() as output_file:
    
    1523 1531
     
    
    1532
    +            if not self.__sandbox_config_supported:
    
    1533
    +                self.warn("Sandbox configuration is not supported by the platform.",
    
    1534
    +                          detail="Falling back to UID {} GID {}. Artifact will not be pushed."
    
    1535
    +                          .format(self.__sandbox_config.build_uid, self.__sandbox_config.build_gid))
    
    1536
    +
    
    1524 1537
                 # Explicitly clean it up, keep the build dir around if exceptions are raised
    
    1525 1538
                 os.makedirs(context.builddir, exist_ok=True)
    
    1526 1539
                 rootdir = tempfile.mkdtemp(prefix="{}-".format(self.normal_name), dir=context.builddir)
    
    ... ... @@ -2110,10 +2123,19 @@ class Element(Plugin):
    2110 2123
                 workspaced_dependencies = self.__get_artifact_metadata_workspaced_dependencies()
    
    2111 2124
     
    
    2112 2125
                 # Other conditions should be or-ed
    
    2113
    -            self.__tainted = workspaced or workspaced_dependencies
    
    2126
    +            self.__tainted = (workspaced or workspaced_dependencies or
    
    2127
    +                              not self.__sandbox_config_supported)
    
    2114 2128
     
    
    2115 2129
             return self.__tainted
    
    2116 2130
     
    
    2131
    +    # __use_remote_execution():
    
    2132
    +    #
    
    2133
    +    # Returns True if remote execution is configured and the element plugin
    
    2134
    +    # supports it.
    
    2135
    +    #
    
    2136
    +    def __use_remote_execution(self):
    
    2137
    +        return self.__remote_execution_url and self.BST_VIRTUAL_DIRECTORY
    
    2138
    +
    
    2117 2139
         # __sandbox():
    
    2118 2140
         #
    
    2119 2141
         # A context manager to prepare a Sandbox object at the specified directory,
    
    ... ... @@ -2135,9 +2157,7 @@ class Element(Plugin):
    2135 2157
             project = self._get_project()
    
    2136 2158
             platform = Platform.get_platform()
    
    2137 2159
     
    
    2138
    -        if (directory is not None and
    
    2139
    -            self.__remote_execution_url and
    
    2140
    -            self.BST_VIRTUAL_DIRECTORY):
    
    2160
    +        if directory is not None and self.__use_remote_execution():
    
    2141 2161
     
    
    2142 2162
                 self.info("Using a remote sandbox for artifact {} with directory '{}'".format(self.name, directory))
    
    2143 2163
     
    

  • buildstream/sandbox/_sandboxremote.py
    ... ... @@ -27,7 +27,6 @@ from . import Sandbox
    27 27
     from ..storage._filebaseddirectory import FileBasedDirectory
    
    28 28
     from ..storage._casbaseddirectory import CasBasedDirectory
    
    29 29
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    
    30
    -from .._platform import Platform
    
    31 30
     
    
    32 31
     
    
    33 32
     class SandboxError(Exception):
    
    ... ... @@ -71,8 +70,8 @@ class SandboxRemote(Sandbox):
    71 70
                                                           output_files=[],
    
    72 71
                                                           output_directories=[self._output_directory],
    
    73 72
                                                           platform=None)
    
    74
    -        platform = Platform.get_platform()
    
    75
    -        cascache = platform.artifactcache
    
    73
    +        context = self._get_context()
    
    74
    +        cascache = context.artifactcache
    
    76 75
             # Upload the Command message to the remote CAS server
    
    77 76
             command_digest = cascache.push_message(self._get_project(), remote_command)
    
    78 77
             if not command_digest or not cascache.verify_digest_pushed(self._get_project(), command_digest):
    
    ... ... @@ -134,8 +133,8 @@ class SandboxRemote(Sandbox):
    134 133
             if tree_digest is None or not tree_digest.hash:
    
    135 134
                 raise SandboxError("Output directory structure had no digest attached.")
    
    136 135
     
    
    137
    -        platform = Platform.get_platform()
    
    138
    -        cascache = platform.artifactcache
    
    136
    +        context = self._get_context()
    
    137
    +        cascache = context.artifactcache
    
    139 138
             # Now do a pull to ensure we have the necessary parts.
    
    140 139
             dir_digest = cascache.pull_tree(self._get_project(), tree_digest)
    
    141 140
             if dir_digest is None or not dir_digest.hash or not dir_digest.size_bytes:
    
    ... ... @@ -170,8 +169,8 @@ class SandboxRemote(Sandbox):
    170 169
     
    
    171 170
             upload_vdir.recalculate_hash()
    
    172 171
     
    
    173
    -        platform = Platform.get_platform()
    
    174
    -        cascache = platform.artifactcache
    
    172
    +        context = self._get_context()
    
    173
    +        cascache = context.artifactcache
    
    175 174
             # Now, push that key (without necessarily needing a ref) to the remote.
    
    176 175
             cascache.push_directory(self._get_project(), upload_vdir)
    
    177 176
             if not cascache.verify_digest_pushed(self._get_project(), upload_vdir.ref):
    

  • buildstream/storage/_casbaseddirectory.py
    ... ... @@ -38,7 +38,6 @@ from .._exceptions import BstError
    38 38
     from .directory import Directory, VirtualDirectoryError
    
    39 39
     from ._filebaseddirectory import FileBasedDirectory
    
    40 40
     from ..utils import FileListResult, safe_copy, list_relative_paths
    
    41
    -from .._artifactcache.cascache import CASCache
    
    42 41
     
    
    43 42
     
    
    44 43
     class IndexEntry():
    
    ... ... @@ -80,7 +79,7 @@ class CasBasedDirectory(Directory):
    80 79
             self.filename = filename
    
    81 80
             self.common_name = common_name
    
    82 81
             self.pb2_directory = remote_execution_pb2.Directory()
    
    83
    -        self.cas_cache = CASCache(context)
    
    82
    +        self.cas_cache = context.artifactcache
    
    84 83
             if ref:
    
    85 84
                 with open(self.cas_cache.objpath(ref), 'rb') as f:
    
    86 85
                     self.pb2_directory.ParseFromString(f.read())
    

  • tests/artifactcache/pull.py
    ... ... @@ -6,7 +6,6 @@ import signal
    6 6
     import pytest
    
    7 7
     
    
    8 8
     from buildstream import _yaml, _signals, utils
    
    9
    -from buildstream._artifactcache.cascache import CASCache
    
    10 9
     from buildstream._context import Context
    
    11 10
     from buildstream._project import Project
    
    12 11
     from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    
    ... ... @@ -88,7 +87,7 @@ def test_pull(cli, tmpdir, datafiles):
    88 87
             # Load the project and CAS cache
    
    89 88
             project = Project(project_dir, context)
    
    90 89
             project.ensure_fully_loaded()
    
    91
    -        cas = CASCache(context)
    
    90
    +        cas = context.artifactcache
    
    92 91
     
    
    93 92
             # Assert that the element's artifact is **not** cached
    
    94 93
             element = project.load_elements(['target.bst'], cas)[0]
    
    ... ... @@ -130,7 +129,7 @@ def _test_pull(user_config_file, project_dir, artifact_dir,
    130 129
         project.ensure_fully_loaded()
    
    131 130
     
    
    132 131
         # Create a local CAS cache handle
    
    133
    -    cas = CASCache(context)
    
    132
    +    cas = context.artifactcache
    
    134 133
     
    
    135 134
         # Load the target element
    
    136 135
         element = project.load_elements([element_name], cas)[0]
    
    ... ... @@ -191,7 +190,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
    191 190
             # Load the project and CAS cache
    
    192 191
             project = Project(project_dir, context)
    
    193 192
             project.ensure_fully_loaded()
    
    194
    -        cas = CASCache(context)
    
    193
    +        cas = context.artifactcache
    
    195 194
     
    
    196 195
             # Assert that the element's artifact is cached
    
    197 196
             element = project.load_elements(['target.bst'], cas)[0]
    
    ... ... @@ -269,7 +268,7 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest
    269 268
         project.ensure_fully_loaded()
    
    270 269
     
    
    271 270
         # Create a local CAS cache handle
    
    272
    -    cas = CASCache(context)
    
    271
    +    cas = context.artifactcache
    
    273 272
     
    
    274 273
         # Manually setup the CAS remote
    
    275 274
         cas.setup_remotes(use_config=True)
    
    ... ... @@ -304,7 +303,7 @@ def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest
    304 303
         project.ensure_fully_loaded()
    
    305 304
     
    
    306 305
         # Create a local CAS cache handle
    
    307
    -    cas = CASCache(context)
    
    306
    +    cas = context.artifactcache
    
    308 307
     
    
    309 308
         # Manually setup the CAS remote
    
    310 309
         cas.setup_remotes(use_config=True)
    

  • tests/artifactcache/push.py
    ... ... @@ -6,7 +6,6 @@ import pytest
    6 6
     
    
    7 7
     from pluginbase import PluginBase
    
    8 8
     from buildstream import _yaml, _signals, utils
    
    9
    -from buildstream._artifactcache.cascache import CASCache
    
    10 9
     from buildstream._context import Context
    
    11 10
     from buildstream._project import Project
    
    12 11
     from buildstream._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    
    ... ... @@ -67,7 +66,7 @@ def test_push(cli, tmpdir, datafiles):
    67 66
             project.ensure_fully_loaded()
    
    68 67
     
    
    69 68
             # Create a local CAS cache handle
    
    70
    -        cas = CASCache(context)
    
    69
    +        cas = context.artifactcache
    
    71 70
     
    
    72 71
             # Assert that the element's artifact is cached
    
    73 72
             element = project.load_elements(['target.bst'], cas)[0]
    
    ... ... @@ -109,7 +108,7 @@ def _test_push(user_config_file, project_dir, artifact_dir,
    109 108
         project.ensure_fully_loaded()
    
    110 109
     
    
    111 110
         # Create a local CAS cache handle
    
    112
    -    cas = CASCache(context)
    
    111
    +    cas = context.artifactcache
    
    113 112
     
    
    114 113
         # Load the target element
    
    115 114
         element = project.load_elements([element_name], cas)[0]
    
    ... ... @@ -166,7 +165,7 @@ def test_push_directory(cli, tmpdir, datafiles):
    166 165
             # Load the project and CAS cache
    
    167 166
             project = Project(project_dir, context)
    
    168 167
             project.ensure_fully_loaded()
    
    169
    -        cas = CASCache(context)
    
    168
    +        cas = context.artifactcache
    
    170 169
     
    
    171 170
             # Assert that the element's artifact is cached
    
    172 171
             element = project.load_elements(['target.bst'], cas)[0]
    
    ... ... @@ -217,7 +216,7 @@ def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_d
    217 216
         project.ensure_fully_loaded()
    
    218 217
     
    
    219 218
         # Create a local CAS cache handle
    
    220
    -    cas = CASCache(context)
    
    219
    +    cas = context.artifactcache
    
    221 220
     
    
    222 221
         # Manually setup the CAS remote
    
    223 222
         cas.setup_remotes(use_config=True)
    
    ... ... @@ -292,7 +291,7 @@ def _test_push_message(user_config_file, project_dir, artifact_dir, queue):
    292 291
         project.ensure_fully_loaded()
    
    293 292
     
    
    294 293
         # Create a local CAS cache handle
    
    295
    -    cas = CASCache(context)
    
    294
    +    cas = context.artifactcache
    
    296 295
     
    
    297 296
         # Manually setup the CAS remote
    
    298 297
         cas.setup_remotes(use_config=True)
    

  • tests/testutils/artifactshare.py
    ... ... @@ -11,7 +11,6 @@ from multiprocessing import Process, Queue
    11 11
     import pytest_cov
    
    12 12
     
    
    13 13
     from buildstream import _yaml
    
    14
    -from buildstream._artifactcache.cascache import CASCache
    
    15 14
     from buildstream._artifactcache.casserver import create_server
    
    16 15
     from buildstream._context import Context
    
    17 16
     from buildstream._exceptions import ArtifactError
    
    ... ... @@ -49,7 +48,7 @@ class ArtifactShare():
    49 48
             context = Context()
    
    50 49
             context.artifactdir = self.repodir
    
    51 50
     
    
    52
    -        self.cas = CASCache(context)
    
    51
    +        self.cas = context.artifactcache
    
    53 52
     
    
    54 53
             self.total_space = total_space
    
    55 54
             self.free_space = free_space
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]