[Notes] [Git][BuildStream/buildstream][lachlan/pickle-yaml-test-list-composite] 19 commits: doc: Add new document about Docker integrations



Title: GitLab

Lachlan pushed to branch lachlan/pickle-yaml-test-list-composite at BuildStream / buildstream

Commits:

17 changed files:

Changes:

  • NEWS
    ... ... @@ -24,6 +24,9 @@ buildstream 1.3.1
    24 24
       o Add new `pip` source plugin for downloading python packages using pip,
    
    25 25
         based on requirements files from previous sources.
    
    26 26
     
    
    27
    +  o Generate Docker images from built artifacts using
    
    28
    +    `contrib/bst-docker-import` script.
    
    29
    +
    
    27 30
     
    
    28 31
     =================
    
    29 32
     buildstream 1.1.5
    

  • README.rst
    ... ... @@ -19,7 +19,8 @@ About
    19 19
     
    
    20 20
     What is BuildStream?
    
    21 21
     ====================
    
    22
    -BuildStream is a Free Software tool for building/integrating software stacks.
    
    22
    +`BuildStream <https://buildstream.build>`_ is a Free Software tool for 
    
    23
    +building/integrating software stacks.
    
    23 24
     It takes inspiration, lessons and use-cases from various projects including
    
    24 25
     OBS, Reproducible Builds, Yocto, Baserock, Buildroot, Aboriginal, GNOME Continuous,
    
    25 26
     JHBuild, Flatpak Builder and Android repo.
    

  • buildstream/__init__.py
    ... ... @@ -28,9 +28,9 @@ if "_BST_COMPLETION" not in os.environ:
    28 28
     
    
    29 29
         from .utils import UtilError, ProgramNotFoundError
    
    30 30
         from .sandbox import Sandbox, SandboxFlags
    
    31
    +    from .types import Scope, Consistency
    
    31 32
         from .plugin import Plugin
    
    32
    -    from .source import Source, SourceError, Consistency, SourceFetcher
    
    33
    +    from .source import Source, SourceError, SourceFetcher
    
    33 34
         from .element import Element, ElementError
    
    34
    -    from .element_enums import Scope
    
    35 35
         from .buildelement import BuildElement
    
    36 36
         from .scriptelement import ScriptElement

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -21,7 +21,7 @@ import os
    21 21
     import string
    
    22 22
     from collections import Mapping, namedtuple
    
    23 23
     
    
    24
    -from ..element_enums import _KeyStrength
    
    24
    +from ..types import _KeyStrength
    
    25 25
     from .._exceptions import ArtifactError, ImplError, LoadError, LoadErrorReason
    
    26 26
     from .._message import Message, MessageType
    
    27 27
     from .. import utils
    

  • buildstream/_loader/loadelement.py
    ... ... @@ -185,6 +185,6 @@ def _extract_depends_from_node(node, *, key=None):
    185 185
             output_deps.append(dependency)
    
    186 186
     
    
    187 187
         # Now delete the field, we dont want it anymore
    
    188
    -    del node[key]
    
    188
    +    node.pop(key, None)
    
    189 189
     
    
    190 190
         return output_deps

  • buildstream/_loader/loader.py
    ... ... @@ -29,6 +29,7 @@ from .. import _yaml
    29 29
     from ..element import Element
    
    30 30
     from .._profile import Topics, profile_start, profile_end
    
    31 31
     from .._includes import Includes
    
    32
    +from .._yamlcache import YamlCache
    
    32 33
     
    
    33 34
     from .types import Symbol, Dependency
    
    34 35
     from .loadelement import LoadElement
    
    ... ... @@ -108,13 +109,19 @@ class Loader():
    108 109
             #
    
    109 110
             deps = []
    
    110 111
     
    
    111
    -        for target in targets:
    
    112
    -            profile_start(Topics.LOAD_PROJECT, target)
    
    113
    -            junction, name, loader = self._parse_name(target, rewritable, ticker,
    
    114
    -                                                      fetch_subprojects=fetch_subprojects)
    
    115
    -            loader._load_file(name, rewritable, ticker, fetch_subprojects)
    
    116
    -            deps.append(Dependency(name, junction=junction))
    
    117
    -            profile_end(Topics.LOAD_PROJECT, target)
    
    112
    +        # XXX This will need to be changed to the context's top-level project if this method
    
    113
    +        # is ever used for subprojects
    
    114
    +        top_dir = self.project.directory
    
    115
    +
    
    116
    +        cache_file = YamlCache.get_cache_file(top_dir)
    
    117
    +        with YamlCache.open(self._context, cache_file) as yaml_cache:
    
    118
    +            for target in targets:
    
    119
    +                profile_start(Topics.LOAD_PROJECT, target)
    
    120
    +                junction, name, loader = self._parse_name(target, rewritable, ticker,
    
    121
    +                                                          fetch_subprojects=fetch_subprojects)
    
    122
    +                loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    
    123
    +                deps.append(Dependency(name, junction=junction))
    
    124
    +                profile_end(Topics.LOAD_PROJECT, target)
    
    118 125
     
    
    119 126
             #
    
    120 127
             # Now that we've resolve the dependencies, scan them for circular dependencies
    
    ... ... @@ -201,11 +208,12 @@ class Loader():
    201 208
         #    rewritable (bool): Whether we should load in round trippable mode
    
    202 209
         #    ticker (callable): A callback to report loaded filenames to the frontend
    
    203 210
         #    fetch_subprojects (bool): Whether to fetch subprojects while loading
    
    211
    +    #    yaml_cache (YamlCache): A yaml cache
    
    204 212
         #
    
    205 213
         # Returns:
    
    206 214
         #    (LoadElement): A loaded LoadElement
    
    207 215
         #
    
    208
    -    def _load_file(self, filename, rewritable, ticker, fetch_subprojects):
    
    216
    +    def _load_file(self, filename, rewritable, ticker, fetch_subprojects, yaml_cache=None):
    
    209 217
     
    
    210 218
             # Silently ignore already loaded files
    
    211 219
             if filename in self._elements:
    
    ... ... @@ -218,7 +226,8 @@ class Loader():
    218 226
             # Load the data and process any conditional statements therein
    
    219 227
             fullpath = os.path.join(self._basedir, filename)
    
    220 228
             try:
    
    221
    -            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
    
    229
    +            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable,
    
    230
    +                              project=self.project, yaml_cache=yaml_cache)
    
    222 231
             except LoadError as e:
    
    223 232
                 if e.reason == LoadErrorReason.MISSING_FILE:
    
    224 233
                     # If we can't find the file, try to suggest plausible
    
    ... ... @@ -261,13 +270,13 @@ class Loader():
    261 270
             # Load all dependency files for the new LoadElement
    
    262 271
             for dep in element.deps:
    
    263 272
                 if dep.junction:
    
    264
    -                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects)
    
    273
    +                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    
    265 274
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    
    266 275
                                               fetch_subprojects=fetch_subprojects)
    
    267 276
                 else:
    
    268 277
                     loader = self
    
    269 278
     
    
    270
    -            dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects)
    
    279
    +            dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects, yaml_cache)
    
    271 280
     
    
    272 281
                 if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
    
    273 282
                     raise LoadError(LoadErrorReason.INVALID_DATA,
    

  • buildstream/_project.py
    ... ... @@ -19,6 +19,7 @@
    19 19
     #        Tiago Gomes <tiago gomes codethink co uk>
    
    20 20
     
    
    21 21
     import os
    
    22
    +import hashlib
    
    22 23
     from collections import Mapping, OrderedDict
    
    23 24
     from pluginbase import PluginBase
    
    24 25
     from . import utils
    

  • buildstream/_yaml.py
    ... ... @@ -183,20 +183,32 @@ class CompositeTypeError(CompositeError):
    183 183
     #    shortname (str): The filename in shorthand for error reporting (or None)
    
    184 184
     #    copy_tree (bool): Whether to make a copy, preserving the original toplevels
    
    185 185
     #                      for later serialization
    
    186
    +#    yaml_cache (YamlCache): A yaml cache to consult rather than parsing
    
    186 187
     #
    
    187 188
     # Returns (dict): A loaded copy of the YAML file with provenance information
    
    188 189
     #
    
    189 190
     # Raises: LoadError
    
    190 191
     #
    
    191
    -def load(filename, shortname=None, copy_tree=False, *, project=None):
    
    192
    +def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=None):
    
    192 193
         if not shortname:
    
    193 194
             shortname = filename
    
    194 195
     
    
    195 196
         file = ProvenanceFile(filename, shortname, project)
    
    196 197
     
    
    197 198
         try:
    
    199
    +        data = None
    
    198 200
             with open(filename) as f:
    
    199
    -            return load_data(f, file, copy_tree=copy_tree)
    
    201
    +            contents = f.read()
    
    202
    +        if yaml_cache:
    
    203
    +            data, key = yaml_cache.get(project, filename, contents, copy_tree)
    
    204
    +
    
    205
    +        if not data:
    
    206
    +            data = load_data(contents, file, copy_tree=copy_tree)
    
    207
    +
    
    208
    +        if yaml_cache:
    
    209
    +            yaml_cache.put_from_key(project, filename, key, data)
    
    210
    +
    
    211
    +        return data
    
    200 212
         except FileNotFoundError as e:
    
    201 213
             raise LoadError(LoadErrorReason.MISSING_FILE,
    
    202 214
                             "Could not find file at {}".format(filename)) from e
    

  • buildstream/_yamlcache.py
    1
    +#
    
    2
    +#  Copyright 2018 Bloomberg Finance LP
    
    3
    +#
    
    4
    +#  This program is free software; you can redistribute it and/or
    
    5
    +#  modify it under the terms of the GNU Lesser General Public
    
    6
    +#  License as published by the Free Software Foundation; either
    
    7
    +#  version 2 of the License, or (at your option) any later version.
    
    8
    +#
    
    9
    +#  This library is distributed in the hope that it will be useful,
    
    10
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    
    11
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    
    12
    +#  Lesser General Public License for more details.
    
    13
    +#
    
    14
    +#  You should have received a copy of the GNU Lesser General Public
    
    15
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    
    16
    +#
    
    17
    +#  Authors:
    
    18
    +#        Jonathan Maw <jonathan maw codethink co uk>
    
    19
    +
    
    20
    +import os
    
    21
    +import pickle
    
    22
    +import hashlib
    
    23
    +import io
    
    24
    +
    
    25
    +import sys
    
    26
    +
    
    27
    +from contextlib import contextmanager
    
    28
    +from collections import namedtuple
    
    29
    +
    
    30
    +from ._cachekey import generate_key
    
    31
    +from ._context import Context
    
    32
    +from . import utils, _yaml
    
    33
    +
    
    34
    +
    
    35
    +YAML_CACHE_FILENAME = "yaml_cache.pickle"
    
    36
    +
    
    37
    +
    
    38
    +# YamlCache()
    
    39
    +#
    
    40
    +# A cache that wraps around the loading of yaml in projects.
    
    41
    +#
    
    42
    +# The recommended way to use a YamlCache is:
    
    43
    +#   with YamlCache.open(context) as yamlcache:
    
    44
    +#     # Load all the yaml
    
    45
    +#     ...
    
    46
    +#
    
    47
    +# Args:
    
    48
    +#    context (Context): The invocation Context
    
    49
    +#
    
    50
    +class YamlCache():
    
    51
    +
    
    52
    +    def __init__(self, context):
    
    53
    +        self._project_caches = {}
    
    54
    +        self._context = context
    
    55
    +
    
    56
    +    ##################
    
    57
    +    # Public Methods #
    
    58
    +    ##################
    
    59
    +
    
    60
    +    # is_cached():
    
    61
    +    #
    
    62
    +    # Checks whether a file is cached.
    
    63
    +    #
    
    64
    +    # Args:
    
    65
    +    #    project (Project): The project this file is in.
    
    66
    +    #    filepath (str): The path to the file, *relative to the project's directory*.
    
    67
    +    #
    
    68
    +    # Returns:
    
    69
    +    #    (bool): Whether the file is cached.
    
    70
    +    def is_cached(self, project, filepath):
    
    71
    +        cache_path = self._get_filepath(project, filepath)
    
    72
    +        project_name = project.name if project else ""
    
    73
    +        try:
    
    74
    +            project_cache = self._project_caches[project_name]
    
    75
    +            if cache_path in project_cache.elements:
    
    76
    +                return True
    
    77
    +        except KeyError:
    
    78
    +            pass
    
    79
    +        return False
    
    80
    +
    
    81
    +    # open():
    
    82
    +    #
    
    83
    +    # Return an instance of the YamlCache which writes to disk when it leaves scope.
    
    84
    +    #
    
    85
    +    # Args:
    
    86
    +    #    context (Context): The context.
    
    87
    +    #    cachefile (str): The path to the cache file.
    
    88
    +    #
    
    89
    +    # Returns:
    
    90
    +    #    (YamlCache): A YamlCache.
    
    91
    +    @staticmethod
    
    92
    +    @contextmanager
    
    93
    +    def open(context, cachefile):
    
    94
    +        # Try to load from disk first
    
    95
    +        cache = None
    
    96
    +        if os.path.exists(cachefile):
    
    97
    +            try:
    
    98
    +                with open(cachefile, "rb") as f:
    
    99
    +                    cache = BstUnpickler(f, context).load()
    
    100
    +            except EOFError:
    
    101
    +                # The file was empty
    
    102
    +                pass
    
    103
    +            except pickle.UnpicklingError as e:
    
    104
    +                sys.stderr.write("Failed to load YamlCache, {}\n".format(e))
    
    105
    +
    
    106
    +        # Failed to load from disk, create a new one
    
    107
    +        if not cache:
    
    108
    +            cache = YamlCache(context)
    
    109
    +
    
    110
    +        yield cache
    
    111
    +
    
    112
    +        cache._write(cachefile)
    
    113
    +
    
    114
    +    # get_cache_file():
    
    115
    +    #
    
    116
    +    # Retrieves a path to the yaml cache file.
    
    117
    +    #
    
    118
    +    # Returns:
    
    119
    +    #   (str): The path to the cache file
    
    120
    +    @staticmethod
    
    121
    +    def get_cache_file(top_dir):
    
    122
    +        return os.path.join(top_dir, ".bst", YAML_CACHE_FILENAME)
    
    123
    +
    
    124
    +    # get():
    
    125
    +    #
    
    126
    +    # Gets a parsed file from the cache.
    
    127
    +    #
    
    128
    +    # Args:
    
    129
    +    #    project (Project) or None: The project this file is in, if it exists.
    
    130
    +    #    filepath (str): The absolute path to the file.
    
    131
    +    #    contents (str): The contents of the file to be cached
    
    132
    +    #    copy_tree (bool): Whether the data should make a copy when it's being generated
    
    133
    +    #                      (i.e. exactly as when called in yaml)
    
    134
    +    #
    
    135
    +    # Returns:
    
    136
    +    #    (decorated dict): The parsed yaml from the cache, or None if the file isn't in the cache.
    
    137
    +    #    (str):            The key used to look up the parsed yaml in the cache
    
    138
    +    def get(self, project, filepath, contents, copy_tree):
    
    139
    +        key = self._calculate_key(contents, copy_tree)
    
    140
    +        data = self._get(project, filepath, key)
    
    141
    +        return data, key
    
    142
    +
    
    143
    +    # put():
    
    144
    +    #
    
    145
    +    # Puts a parsed file into the cache.
    
    146
    +    #
    
    147
    +    # Args:
    
    148
    +    #    project (Project): The project this file is in.
    
    149
    +    #    filepath (str): The path to the file.
    
    150
    +    #    contents (str): The contents of the file that has been cached
    
    151
    +    #    copy_tree (bool): Whether the data should make a copy when it's being generated
    
    152
    +    #                      (i.e. exactly as when called in yaml)
    
    153
    +    #    value (decorated dict): The data to put into the cache.
    
    154
    +    def put(self, project, filepath, contents, copy_tree, value):
    
    155
    +        key = self._calculate_key(contents, copy_tree)
    
    156
    +        self.put_from_key(project, filepath, key, value)
    
    157
    +
    
    158
    +    # put_from_key():
    
    159
    +    #
    
    160
    +    # Put a parsed file into the cache when given a key.
    
    161
    +    #
    
    162
    +    # Args:
    
    163
    +    #    project (Project): The project this file is in.
    
    164
    +    #    filepath (str): The path to the file.
    
    165
    +    #    key (str): The key to the file within the cache. Typically, this is the
    
    166
    +    #               value of `calculate_key()` with the file's unparsed contents
    
    167
    +    #               and any relevant metadata passed in.
    
    168
    +    #    value (decorated dict): The data to put into the cache.
    
    169
    +    def put_from_key(self, project, filepath, key, value):
    
    170
    +        cache_path = self._get_filepath(project, filepath)
    
    171
    +        project_name = project.name if project else ""
    
    172
    +        try:
    
    173
    +            project_cache = self._project_caches[project_name]
    
    174
    +        except KeyError:
    
    175
    +            project_cache = self._project_caches[project_name] = CachedProject({})
    
    176
    +
    
    177
    +        project_cache.elements[cache_path] = CachedYaml(key, value)
    
    178
    +
    
    179
    +    ###################
    
    180
    +    # Private Methods #
    
    181
    +    ###################
    
    182
    +
    
    183
    +    # Writes the yaml cache to the specified path.
    
    184
    +    #
    
    185
    +    # Args:
    
    186
    +    #    path (str): The path to the cache file.
    
    187
    +    def _write(self, path):
    
    188
    +        parent_dir = os.path.dirname(path)
    
    189
    +        os.makedirs(parent_dir, exist_ok=True)
    
    190
    +        with open(path, "wb") as f:
    
    191
    +            BstPickler(f).dump(self)
    
    192
    +
    
    193
    +    # _get_filepath():
    
    194
    +    #
    
    195
    +    # Returns a file path relative to a project if passed, or the original path if
    
    196
    +    # the project is None
    
    197
    +    #
    
    198
    +    # Args:
    
    199
    +    #    project (Project) or None: The project the filepath exists within
    
    200
    +    #    full_path (str): The path that the returned path is based on
    
    201
    +    #
    
    202
    +    # Returns:
    
    203
    +    #    (str): The path to the file, relative to a project if it exists
    
    204
    +    def _get_filepath(self, project, full_path):
    
    205
    +        if project:
    
    206
    +            assert full_path.startswith(project.directory)
    
    207
    +            filepath = os.path.relpath(full_path, project.directory)
    
    208
    +        else:
    
    209
    +            filepath = full_path
    
    210
    +        return full_path
    
    211
    +
    
    212
    +    # _calculate_key():
    
    213
    +    #
    
    214
    +    # Calculates a key for putting into the cache.
    
    215
    +    #
    
    216
    +    # Args:
    
    217
    +    #    (basic object)... : Any number of strictly-ordered basic objects
    
    218
    +    #
    
    219
    +    # Returns:
    
    220
    +    #   (str): A key made out of every arg passed in
    
    221
    +    @staticmethod
    
    222
    +    def _calculate_key(*args):
    
    223
    +        string = pickle.dumps(args)
    
    224
    +        return hashlib.sha1(string).hexdigest()
    
    225
    +
    
    226
    +    # _get():
    
    227
    +    #
    
    228
    +    # Gets a parsed file from the cache when given a key.
    
    229
    +    #
    
    230
    +    # Args:
    
    231
    +    #    project (Project): The project this file is in.
    
    232
    +    #    filepath (str): The path to the file.
    
    233
    +    #    key (str): The key to the file within the cache. Typically, this is the
    
    234
    +    #               value of `calculate_key()` with the file's unparsed contents
    
    235
    +    #               and any relevant metadata passed in.
    
    236
    +    #
    
    237
    +    # Returns:
    
    238
    +    #    (decorated dict): The parsed yaml from the cache, or None if the file isn't in the cache.
    
    239
    +    def _get(self, project, filepath, key):
    
    240
    +        cache_path = self._get_filepath(project, filepath)
    
    241
    +        project_name = project.name if project else ""
    
    242
    +        try:
    
    243
    +            project_cache = self._project_caches[project_name]
    
    244
    +            try:
    
    245
    +                cachedyaml = project_cache.elements[cache_path]
    
    246
    +                if cachedyaml._key == key:
    
    247
    +                    # We've unpickled the YamlCache, but not the specific file
    
    248
    +                    if cachedyaml._contents is None:
    
    249
    +                        cachedyaml._contents = BstUnpickler.loads(cachedyaml._pickled_contents, self._context)
    
    250
    +                    return cachedyaml._contents
    
    251
    +            except KeyError:
    
    252
    +                pass
    
    253
    +        except KeyError:
    
    254
    +            pass
    
    255
    +        return None
    
    256
    +
    
    257
    +
    
    258
    +CachedProject = namedtuple('CachedProject', ['elements'])
    
    259
    +
    
    260
    +
    
    261
    +class CachedYaml():
    
    262
    +    def __init__(self, key, contents):
    
    263
    +        self._key = key
    
    264
    +        self.set_contents(contents)
    
    265
    +
    
    266
    +    # Sets the contents of the CachedYaml.
    
    267
    +    #
    
    268
    +    # Args:
    
    269
    +    #    contents (provenanced dict): The contents to put in the cache.
    
    270
    +    #
    
    271
    +    def set_contents(self, contents):
    
    272
    +        self._contents = contents
    
    273
    +        self._pickled_contents = BstPickler.dumps(contents)
    
    274
    +
    
    275
    +    # Pickling helper method, prevents 'contents' from being serialised
    
    276
    +    def __getstate__(self):
    
    277
    +        data = self.__dict__.copy()
    
    278
    +        data['_contents'] = None
    
    279
    +        return data
    
    280
    +
    
    281
    +
    
    282
    +# In _yaml.load, we have a ProvenanceFile that stores the project the file
    
    283
    +# came from. Projects can't be pickled, but it's always going to be the same
    
    284
    +# project between invocations (unless the entire project is moved but the
    
    285
    +# file stayed in the same place)
    
    286
    +class BstPickler(pickle.Pickler):
    
    287
    +    def persistent_id(self, obj):
    
    288
    +        if isinstance(obj, _yaml.ProvenanceFile):
    
    289
    +            if obj.project:
    
    290
    +                # ProvenanceFile's project object cannot be stored as it is.
    
    291
    +                project_tag = obj.project.name
    
    292
    +                # ProvenanceFile's filename must be stored relative to the
    
    293
    +                # project, as the project dir may move.
    
    294
    +                name = os.path.relpath(obj.name, obj.project.directory)
    
    295
    +            else:
    
    296
    +                project_tag = None
    
    297
    +                name = obj.name
    
    298
    +            return ("ProvenanceFile", name, obj.shortname, project_tag)
    
    299
    +        elif isinstance(obj, Context):
    
    300
    +            return ("Context",)
    
    301
    +        else:
    
    302
    +            return None
    
    303
    +
    
    304
    +    @staticmethod
    
    305
    +    def dumps(obj):
    
    306
    +        stream = io.BytesIO()
    
    307
    +        BstPickler(stream).dump(obj)
    
    308
    +        stream.seek(0)
    
    309
    +        return stream.read()
    
    310
    +
    
    311
    +
    
    312
    +class BstUnpickler(pickle.Unpickler):
    
    313
    +    def __init__(self, file, context):
    
    314
    +        super().__init__(file)
    
    315
    +        self._context = context
    
    316
    +
    
    317
    +    def persistent_load(self, pid):
    
    318
    +        if pid[0] == "ProvenanceFile":
    
    319
    +            _, tagged_name, shortname, project_tag = pid
    
    320
    +
    
    321
    +            if project_tag is not None:
    
    322
    +                for p in self._context.get_projects():
    
    323
    +                    if project_tag == p.name:
    
    324
    +                        project = p
    
    325
    +                        break
    
    326
    +
    
    327
    +                name = os.path.join(project.directory, tagged_name)
    
    328
    +
    
    329
    +                if not project:
    
    330
    +                    projects = [p.name for p in self._context.get_projects()]
    
    331
    +                    raise pickle.UnpicklingError("No project with name {} found in {}"
    
    332
    +                                                 .format(key_id, projects))
    
    333
    +            else:
    
    334
    +                project = None
    
    335
    +                name = tagged_name
    
    336
    +
    
    337
    +            return _yaml.ProvenanceFile(name, shortname, project)
    
    338
    +        elif pid[0] == "Context":
    
    339
    +            return self._context
    
    340
    +        else:
    
    341
    +            raise pickle.UnpicklingError("Unsupported persistent object, {}".format(pid))
    
    342
    +
    
    343
    +    @staticmethod
    
    344
    +    def loads(text, context):
    
    345
    +        stream = io.BytesIO()
    
    346
    +        stream.write(bytes(text))
    
    347
    +        stream.seek(0)
    
    348
    +        return BstUnpickler(stream, context).load()

  • buildstream/element.py
    ... ... @@ -86,7 +86,7 @@ from ._variables import Variables
    86 86
     from ._versions import BST_CORE_ARTIFACT_VERSION
    
    87 87
     from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
    
    88 88
     from .utils import UtilError
    
    89
    -from . import Plugin, Consistency
    
    89
    +from . import Plugin, Consistency, Scope
    
    90 90
     from . import SandboxFlags
    
    91 91
     from . import utils
    
    92 92
     from . import _cachekey
    
    ... ... @@ -96,11 +96,11 @@ from ._platform import Platform
    96 96
     from .plugin import CoreWarnings
    
    97 97
     from .sandbox._config import SandboxConfig
    
    98 98
     from .sandbox._sandboxremote import SandboxRemote
    
    99
    +from .types import _KeyStrength
    
    99 100
     
    
    100 101
     from .storage.directory import Directory
    
    101 102
     from .storage._filebaseddirectory import FileBasedDirectory
    
    102 103
     from .storage.directory import VirtualDirectoryError
    
    103
    -from .element_enums import _KeyStrength, Scope
    
    104 104
     
    
    105 105
     
    
    106 106
     class ElementError(BstError):
    

  • buildstream/source.py
    ... ... @@ -145,35 +145,12 @@ import os
    145 145
     from collections import Mapping
    
    146 146
     from contextlib import contextmanager
    
    147 147
     
    
    148
    -from . import Plugin
    
    148
    +from . import Plugin, Consistency
    
    149 149
     from . import _yaml, utils
    
    150 150
     from ._exceptions import BstError, ImplError, ErrorDomain
    
    151 151
     from ._projectrefs import ProjectRefStorage
    
    152 152
     
    
    153 153
     
    
    154
    -class Consistency():
    
    155
    -    INCONSISTENT = 0
    
    156
    -    """Inconsistent
    
    157
    -
    
    158
    -    Inconsistent sources have no explicit reference set. They cannot
    
    159
    -    produce a cache key, be fetched or staged. They can only be tracked.
    
    160
    -    """
    
    161
    -
    
    162
    -    RESOLVED = 1
    
    163
    -    """Resolved
    
    164
    -
    
    165
    -    Resolved sources have a reference and can produce a cache key and
    
    166
    -    be fetched, however they cannot be staged.
    
    167
    -    """
    
    168
    -
    
    169
    -    CACHED = 2
    
    170
    -    """Cached
    
    171
    -
    
    172
    -    Cached sources have a reference which is present in the local
    
    173
    -    source cache. Only cached sources can be staged.
    
    174
    -    """
    
    175
    -
    
    176
    -
    
    177 154
     class SourceError(BstError):
    
    178 155
         """This exception should be raised by :class:`.Source` implementations
    
    179 156
         to report errors to the user.
    

  • buildstream/element_enums.pybuildstream/types.py
    ... ... @@ -19,31 +19,19 @@
    19 19
     #        Jim MacArthur <jim macarthur codethink co uk>
    
    20 20
     
    
    21 21
     """
    
    22
    -Element - Globally visible enumerations
    
    23
    -=======================================
    
    22
    +Foundation types
    
    23
    +================
    
    24 24
     
    
    25 25
     """
    
    26 26
     
    
    27 27
     from enum import Enum
    
    28 28
     
    
    29 29
     
    
    30
    -# _KeyStrength():
    
    31
    -#
    
    32
    -# Strength of cache key
    
    33
    -#
    
    34
    -class _KeyStrength(Enum):
    
    35
    -
    
    36
    -    # Includes strong cache keys of all build dependencies and their
    
    37
    -    # runtime dependencies.
    
    38
    -    STRONG = 1
    
    39
    -
    
    40
    -    # Includes names of direct build dependencies but does not include
    
    41
    -    # cache keys of dependencies.
    
    42
    -    WEAK = 2
    
    43
    -
    
    44
    -
    
    45 30
     class Scope(Enum):
    
    46
    -    """Types of scope for a given element"""
    
    31
    +    """Defines the scope of dependencies to include for a given element
    
    32
    +    when iterating over the dependency graph in APIs like
    
    33
    +    :func:`Element.dependencies() <buildstream.element.Element.dependencies>`
    
    34
    +    """
    
    47 35
     
    
    48 36
         ALL = 1
    
    49 37
         """All elements which the given element depends on, following
    
    ... ... @@ -59,3 +47,44 @@ class Scope(Enum):
    59 47
         """All elements required for running the element. Including the element
    
    60 48
         itself.
    
    61 49
         """
    
    50
    +
    
    51
    +
    
    52
    +class Consistency():
    
    53
    +    """Defines the various consistency states of a :class:`.Source`.
    
    54
    +    """
    
    55
    +
    
    56
    +    INCONSISTENT = 0
    
    57
    +    """Inconsistent
    
    58
    +
    
    59
    +    Inconsistent sources have no explicit reference set. They cannot
    
    60
    +    produce a cache key, be fetched or staged. They can only be tracked.
    
    61
    +    """
    
    62
    +
    
    63
    +    RESOLVED = 1
    
    64
    +    """Resolved
    
    65
    +
    
    66
    +    Resolved sources have a reference and can produce a cache key and
    
    67
    +    be fetched, however they cannot be staged.
    
    68
    +    """
    
    69
    +
    
    70
    +    CACHED = 2
    
    71
    +    """Cached
    
    72
    +
    
    73
    +    Cached sources have a reference which is present in the local
    
    74
    +    source cache. Only cached sources can be staged.
    
    75
    +    """
    
    76
    +
    
    77
    +
    
    78
    +# _KeyStrength():
    
    79
    +#
    
    80
    +# Strength of cache key
    
    81
    +#
    
    82
    +class _KeyStrength(Enum):
    
    83
    +
    
    84
    +    # Includes strong cache keys of all build dependencies and their
    
    85
    +    # runtime dependencies.
    
    86
    +    STRONG = 1
    
    87
    +
    
    88
    +    # Includes names of direct build dependencies but does not include
    
    89
    +    # cache keys of dependencies.
    
    90
    +    WEAK = 2

  • doc/source/additional_docker.rst
    1
    +
    
    2
    +.. _bst_and_docker:
    
    3
    +
    
    4
    +
    
    5
    +BuildStream and Docker
    
    6
    +======================
    
    7
    +
    
    8
    +BuildStream integrates with Docker in multiple ways. Here are some ways in
    
    9
    +which these integrations work.
    
    10
    +
    
    11
    +Run BuildStream inside Docker
    
    12
    +-----------------------------
    
    13
    +
    
    14
    +Refer to the :ref:`BuildStream inside Docker <docker>` documentation for
    
    15
    +instructions on how to run BuildStream as a Docker container.
    
    16
    +
    
    17
    +Generate Docker images
    
    18
    +----------------------
    
    19
    +
    
    20
    +The
    
    21
    +`bst-docker-import script <https://gitlab.com/BuildStream/buildstream/blob/master/contrib/bst-docker-import>`_
    
    22
    +can be used to generate a Docker image from built artifacts.
    
    23
    +
    
    24
    +You can download it and make it executable like this:
    
    25
    +
    
    26
    +.. code:: bash
    
    27
    +
    
    28
    +  mkdir -p ~/.local/bin
    
    29
    +  curl --get https://gitlab.com/BuildStream/buildstream/raw/master/contrib/bst-docker-import > ~/.local/bin/bst-docker-import
    
    30
    +  chmod +x ~/.local/bin/bst-docker-import
    
    31
    +
    
    32
    +Check if ``~/.local/bin`` appears in your PATH environment variable -- if it
    
    33
    +doesn't, you should
    
    34
    +`edit your ~/.profile so that it does <https://stackoverflow.com/questions/14637979/>`_.
    
    35
    +
    
    36
    +Once the script is available in your PATH and assuming you have Docker
    
    37
    +installed, you can start using the ``bst-docker-import`` script. Here is a
    
    38
    +minimal example to generate an image called ``bst-hello`` from an element
    
    39
    +called ``hello.bst`` assuming it is already built:
    
    40
    +
    
    41
    +.. code:: bash
    
    42
    +
    
    43
    +  bst-docker-import -t bst-hello hello.bst
    
    44
    +
    
    45
    +This script can also be used if you are running BuildStream inside Docker. In
    
    46
    +this case, you will need to supply the command that you are using to run
    
    47
    +BuildStream using the ``-c`` option.  If you are using the
    
    48
    +`bst-here wrapper script <https://gitlab.com/BuildStream/buildstream/blob/master/contrib/bst-here>`_,
    
    49
    +you can achieve the same results as the above example like this:
    
    50
    +
    
    51
    +.. code:: bash
    
    52
    +
    
    53
    +  bst-docker-import -c bst-here -t bst-hello hello.bst

  • doc/source/core_additional.rst
    ... ... @@ -8,3 +8,4 @@ Additional writings
    8 8
     
    
    9 9
        additional_cachekeys
    
    10 10
        additional_sandboxing
    
    11
    +   additional_docker

  • doc/source/core_framework.rst
    ... ... @@ -12,6 +12,7 @@ useful for working on BuildStream itself.
    12 12
     .. toctree::
    
    13 13
        :maxdepth: 1
    
    14 14
     
    
    15
    +   buildstream.types
    
    15 16
        buildstream.plugin
    
    16 17
        buildstream.source
    
    17 18
        buildstream.element
    

  • tests/frontend/yamlcache.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +import hashlib
    
    4
    +import tempfile
    
    5
    +from ruamel import yaml
    
    6
    +
    
    7
    +from tests.testutils import cli, generate_junction, create_element_size, create_repo
    
    8
    +from buildstream import _yaml
    
    9
    +from buildstream._yamlcache import YamlCache
    
    10
    +from buildstream._project import Project
    
    11
    +from buildstream._context import Context
    
    12
    +from contextlib import contextmanager
    
    13
    +
    
    14
    +
    
    15
    +def generate_project(tmpdir, ref_storage, with_junction, name="test"):
    
    16
    +    if with_junction == 'junction':
    
    17
    +        subproject_dir = generate_project(
    
    18
    +            tmpdir, ref_storage,
    
    19
    +            'no-junction', name='test-subproject'
    
    20
    +        )
    
    21
    +
    
    22
    +    project_dir = os.path.join(tmpdir, name)
    
    23
    +    os.makedirs(project_dir)
    
    24
    +    # project.conf
    
    25
    +    project_conf_path = os.path.join(project_dir, 'project.conf')
    
    26
    +    elements_path = 'elements'
    
    27
    +    project_conf = {
    
    28
    +        'name': name,
    
    29
    +        'element-path': elements_path,
    
    30
    +        'ref-storage': ref_storage,
    
    31
    +    }
    
    32
    +    _yaml.dump(project_conf, project_conf_path)
    
    33
    +
    
    34
    +    # elements
    
    35
    +    if with_junction == 'junction':
    
    36
    +        junction_name = 'junction.bst'
    
    37
    +        junction_dir = os.path.join(project_dir, elements_path)
    
    38
    +        junction_path = os.path.join(project_dir, elements_path, junction_name)
    
    39
    +        os.makedirs(junction_dir)
    
    40
    +        generate_junction(tmpdir, subproject_dir, junction_path)
    
    41
    +        element_depends = [{'junction': junction_name, 'filename': 'test.bst'}]
    
    42
    +    else:
    
    43
    +        element_depends = []
    
    44
    +
    
    45
    +    element_name = 'test.bst'
    
    46
    +    create_element_size(element_name, project_dir, elements_path, element_depends, 1)
    
    47
    +
    
    48
    +    return project_dir
    
    49
    +
    
    50
    +
    
    51
    +@contextmanager
    
    52
    +def with_yamlcache(project_dir):
    
    53
    +    context = Context()
    
    54
    +    project = Project(project_dir, context)
    
    55
    +    cache_file = YamlCache.get_cache_file(project_dir)
    
    56
    +    with YamlCache.open(context, cache_file) as yamlcache:
    
    57
    +        yield yamlcache, project
    
    58
    +
    
    59
    +
    
    60
    +def yamlcache_key(yamlcache, in_file, copy_tree=False):
    
    61
    +    with open(in_file) as f:
    
    62
    +        key = yamlcache._calculate_key(f.read(), copy_tree)
    
    63
    +    return key
    
    64
    +
    
    65
    +
    
    66
    +def modified_file(input_file, tmpdir):
    
    67
    +    with open(input_file) as f:
    
    68
    +        data = f.read()
    
    69
    +    assert 'variables' not in data
    
    70
    +    data += '\nvariables: {modified: True}\n'
    
    71
    +    _, temppath = tempfile.mkstemp(dir=tmpdir, text=True)
    
    72
    +    with open(temppath, 'w') as f:
    
    73
    +        f.write(data)
    
    74
    +
    
    75
    +    return temppath
    
    76
    +
    
    77
    +
    
    78
    +@pytest.mark.parametrize('ref_storage', ['inline', 'project.refs'])
    
    79
    +@pytest.mark.parametrize('with_junction', ['no-junction', 'junction'])
    
    80
    +@pytest.mark.parametrize('move_project', ['move', 'no-move'])
    
    81
    +def test_yamlcache_used(cli, tmpdir, ref_storage, with_junction, move_project):
    
    82
    +    # Generate the project
    
    83
    +    project = generate_project(str(tmpdir), ref_storage, with_junction)
    
    84
    +    if with_junction == 'junction':
    
    85
    +        result = cli.run(project=project, args=['fetch', '--track', 'junction.bst'])
    
    86
    +        result.assert_success()
    
    87
    +
    
    88
    +    # bst show to put it in the cache
    
    89
    +    result = cli.run(project=project, args=['show', 'test.bst'])
    
    90
    +    result.assert_success()
    
    91
    +
    
    92
    +    element_path = os.path.join(project, 'elements', 'test.bst')
    
    93
    +    with with_yamlcache(project) as (yc, prj):
    
    94
    +        # Check that it's in the cache
    
    95
    +        assert yc.is_cached(prj, element_path)
    
    96
    +
    
    97
    +        # *Absolutely* horrible cache corruption to check it's being used
    
    98
    +        # Modifying the data from the cache is fraught with danger,
    
    99
    +        # so instead I'll load a modified version of the original file
    
    100
    +        temppath = modified_file(element_path, str(tmpdir))
    
    101
    +        contents = _yaml.load(temppath, copy_tree=False, project=prj)
    
    102
    +        key = yamlcache_key(yc, element_path)
    
    103
    +        yc.put_from_key(prj, element_path, key, contents)
    
    104
    +
    
    105
    +    # Show that a variable has been added
    
    106
    +    result = cli.run(project=project, args=['show', '--format', '%{vars}', 'test.bst'])
    
    107
    +    result.assert_success()
    
    108
    +    data = yaml.safe_load(result.output)
    
    109
    +    assert 'modified' in data
    
    110
    +    assert data['modified'] == 'True'
    
    111
    +
    
    112
    +
    
    113
    +@pytest.mark.parametrize('ref_storage', ['inline', 'project.refs'])
    
    114
    +@pytest.mark.parametrize('with_junction', ['junction', 'no-junction'])
    
    115
    +@pytest.mark.parametrize('move_project', ['move', 'no-move'])
    
    116
    +def test_yamlcache_changed_file(cli, ref_storage, with_junction, move_project):
    
    117
    +    # i.e. a file is cached, the file is changed, loading the file (with cache) returns new data
    
    118
    +    # inline and junction can only be changed by opening a workspace
    
    119
    +    pass

  • tests/yaml/yaml.py
    1 1
     import os
    
    2 2
     import pytest
    
    3
    +import tempfile
    
    3 4
     from collections import Mapping
    
    4 5
     
    
    5 6
     from buildstream import _yaml
    
    6 7
     from buildstream._exceptions import LoadError, LoadErrorReason
    
    8
    +from buildstream._context import Context
    
    9
    +from buildstream._yamlcache import YamlCache
    
    7 10
     
    
    8 11
     DATA_DIR = os.path.join(
    
    9 12
         os.path.dirname(os.path.realpath(__file__)),
    
    ... ... @@ -150,6 +153,21 @@ def test_composite_preserve_originals(datafiles):
    150 153
         assert(_yaml.node_get(orig_extra, str, 'old') == 'new')
    
    151 154
     
    
    152 155
     
    
    156
    +def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
    
    157
    +
    
    158
    +    _, temppath = tempfile.mkstemp(dir=os.path.join(cache_path.dirname, cache_path.basename), text=True)
    
    159
    +    context = Context()
    
    160
    +
    
    161
    +    with YamlCache.open(context, temppath) as yc:
    
    162
    +        if from_cache == 'raw':
    
    163
    +            return _yaml.load(filename, shortname)
    
    164
    +        elif from_cache == 'cached':
    
    165
    +            _yaml.load(filename, shortname, yaml_cache=yc)
    
    166
    +            return _yaml.load(filename, shortname, yaml_cache=yc)
    
    167
    +        else:
    
    168
    +            assert False
    
    169
    +
    
    170
    +
    
    153 171
     # Tests for list composition
    
    154 172
     #
    
    155 173
     # Each test composits a filename on top of basics.yaml, and tests
    
    ... ... @@ -165,6 +183,7 @@ def test_composite_preserve_originals(datafiles):
    165 183
     #    prov_col: The expected provenance column of "mood"
    
    166 184
     #
    
    167 185
     @pytest.mark.datafiles(os.path.join(DATA_DIR))
    
    186
    +@pytest.mark.parametrize('caching', [('raw'), ('cached')])
    
    168 187
     @pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [
    
    169 188
     
    
    170 189
         # Test results of compositing with the (<) prepend directive
    
    ... ... @@ -195,14 +214,15 @@ def test_composite_preserve_originals(datafiles):
    195 214
         ('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
    
    196 215
         ('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
    
    197 216
     ])
    
    198
    -def test_list_composition(datafiles, filename,
    
    217
    +def test_list_composition(datafiles, filename, tmpdir,
    
    199 218
                               index, length, mood,
    
    200
    -                          prov_file, prov_line, prov_col):
    
    201
    -    base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    
    202
    -    overlay = os.path.join(datafiles.dirname, datafiles.basename, filename)
    
    219
    +                          prov_file, prov_line, prov_col, caching):
    
    220
    +    base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    
    221
    +    overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)
    
    222
    +
    
    223
    +    base = load_yaml_file(base_file, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    
    224
    +    overlay = load_yaml_file(overlay_file, cache_path=tmpdir, shortname=filename, from_cache=caching)
    
    203 225
     
    
    204
    -    base = _yaml.load(base, shortname='basics.yaml')
    
    205
    -    overlay = _yaml.load(overlay, shortname=filename)
    
    206 226
         _yaml.composite_dict(base, overlay)
    
    207 227
     
    
    208 228
         children = _yaml.node_get(base, list, 'children')
    
    ... ... @@ -254,6 +274,7 @@ def test_list_deletion(datafiles):
    254 274
     #    prov_col: The expected provenance column of "mood"
    
    255 275
     #
    
    256 276
     @pytest.mark.datafiles(os.path.join(DATA_DIR))
    
    277
    +@pytest.mark.parametrize('caching', [('raw'), ('cached')])
    
    257 278
     @pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [
    
    258 279
     
    
    259 280
         # Test results of compositing literal list with (>) and then (<)
    
    ... ... @@ -310,9 +331,9 @@ def test_list_deletion(datafiles):
    310 331
         ('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    
    311 332
         ('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
    
    312 333
     ])
    
    313
    -def test_list_composition_twice(datafiles, filename1, filename2,
    
    334
    +def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
    
    314 335
                                     index, length, mood,
    
    315
    -                                prov_file, prov_line, prov_col):
    
    336
    +                                prov_file, prov_line, prov_col, caching):
    
    316 337
         file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    
    317 338
         file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
    
    318 339
         file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)
    
    ... ... @@ -320,9 +341,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
    320 341
         #####################
    
    321 342
         # Round 1 - Fight !
    
    322 343
         #####################
    
    323
    -    base = _yaml.load(file_base, shortname='basics.yaml')
    
    324
    -    overlay1 = _yaml.load(file1, shortname=filename1)
    
    325
    -    overlay2 = _yaml.load(file2, shortname=filename2)
    
    344
    +    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    
    345
    +    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    
    346
    +    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
    
    326 347
     
    
    327 348
         _yaml.composite_dict(base, overlay1)
    
    328 349
         _yaml.composite_dict(base, overlay2)
    
    ... ... @@ -337,9 +358,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
    337 358
         #####################
    
    338 359
         # Round 2 - Fight !
    
    339 360
         #####################
    
    340
    -    base = _yaml.load(file_base, shortname='basics.yaml')
    
    341
    -    overlay1 = _yaml.load(file1, shortname=filename1)
    
    342
    -    overlay2 = _yaml.load(file2, shortname=filename2)
    
    361
    +    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    
    362
    +    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    
    363
    +    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
    
    343 364
     
    
    344 365
         _yaml.composite_dict(overlay1, overlay2)
    
    345 366
         _yaml.composite_dict(base, overlay1)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]