GNOME.org
 

[Notes] [Git][BuildStream/buildstream][lachlan/pickle-yaml-test-list-composite] 19 commits: doc: Add new document about Docker integrations

Title: GitLab

Lachlan pushed to branch lachlan/pickle-yaml-test-list-composite at BuildStream / buildstream

Commits:

17 changed files:

Changes:

  • NEWS
    ... ... @@ -24,6 +24,9 @@ buildstream 1.3.1
    24 24 
       o Add new `pip` source plugin for downloading python packages using pip,
    25 25 
         based on requirements files from previous sources.
    26 26
    27 
    +  o Generate Docker images from built artifacts using
    28 
    +    `contrib/bst-docker-import` script.
    29 
    +
    27 30
    28 31 
     =================
    29 32 
     buildstream 1.1.5

  • README.rst
    ... ... @@ -19,7 +19,8 @@ About
    19 19
    20 20 
     What is BuildStream?
    21 21 
     ====================
    22 
    -BuildStream is a Free Software tool for building/integrating software stacks.
    22 
    +`BuildStream <https://buildstream.build>`_ is a Free Software tool for
    23 
    +building/integrating software stacks.
    23 24 
     It takes inspiration, lessons and use-cases from various projects including
    24 25 
     OBS, Reproducible Builds, Yocto, Baserock, Buildroot, Aboriginal, GNOME Continuous,
    25 26 
     JHBuild, Flatpak Builder and Android repo.

  • buildstream/__init__.py
    ... ... @@ -28,9 +28,9 @@ if "_BST_COMPLETION" not in os.environ:
    28 28
    29 29 
         from .utils import UtilError, ProgramNotFoundError
    30 30 
         from .sandbox import Sandbox, SandboxFlags
    31 
    +    from .types import Scope, Consistency
    31 32 
         from .plugin import Plugin
    32 
    -    from .source import Source, SourceError, Consistency, SourceFetcher
    33 
    +    from .source import Source, SourceError, SourceFetcher
    33 34 
         from .element import Element, ElementError
    34 
    -    from .element_enums import Scope
    35 35 
         from .buildelement import BuildElement
    36 36 
         from .scriptelement import ScriptElement

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -21,7 +21,7 @@ import os
    21 21 
     import string
    22 22 
     from collections import Mapping, namedtuple
    23 23
    24 
    -from ..element_enums import _KeyStrength
    24 
    +from ..types import _KeyStrength
    25 25 
     from .._exceptions import ArtifactError, ImplError, LoadError, LoadErrorReason
    26 26 
     from .._message import Message, MessageType
    27 27 
     from .. import utils

  • buildstream/_loader/loadelement.py
    ... ... @@ -185,6 +185,6 @@ def _extract_depends_from_node(node, *, key=None):
    185 185 
             output_deps.append(dependency)
    186 186
    187 187 
         # Now delete the field, we dont want it anymore
    188 
    -    del node[key]
    188 
    +    node.pop(key, None)
    189 189
    190 190 
         return output_deps

  • buildstream/_loader/loader.py
    ... ... @@ -29,6 +29,7 @@ from .. import _yaml
    29 29 
     from ..element import Element
    30 30 
     from .._profile import Topics, profile_start, profile_end
    31 31 
     from .._includes import Includes
    32 
    +from .._yamlcache import YamlCache
    32 33
    33 34 
     from .types import Symbol, Dependency
    34 35 
     from .loadelement import LoadElement
    ... ... @@ -108,13 +109,19 @@ class Loader():
    108 109 
             #
    109 110 
             deps = []
    110 111
    111 
    -        for target in targets:
    112 
    -            profile_start(Topics.LOAD_PROJECT, target)
    113 
    -            junction, name, loader = self._parse_name(target, rewritable, ticker,
    114 
    -                                                      fetch_subprojects=fetch_subprojects)
    115 
    -            loader._load_file(name, rewritable, ticker, fetch_subprojects)
    116 
    -            deps.append(Dependency(name, junction=junction))
    117 
    -            profile_end(Topics.LOAD_PROJECT, target)
    112 
    +        # XXX This will need to be changed to the context's top-level project if this method
    113 
    +        # is ever used for subprojects
    114 
    +        top_dir = self.project.directory
    115 
    +
    116 
    +        cache_file = YamlCache.get_cache_file(top_dir)
    117 
    +        with YamlCache.open(self._context, cache_file) as yaml_cache:
    118 
    +            for target in targets:
    119 
    +                profile_start(Topics.LOAD_PROJECT, target)
    120 
    +                junction, name, loader = self._parse_name(target, rewritable, ticker,
    121 
    +                                                          fetch_subprojects=fetch_subprojects)
    122 
    +                loader._load_file(name, rewritable, ticker, fetch_subprojects, yaml_cache)
    123 
    +                deps.append(Dependency(name, junction=junction))
    124 
    +                profile_end(Topics.LOAD_PROJECT, target)
    118 125
    119 126 
             #
    120 127 
             # Now that we've resolve the dependencies, scan them for circular dependencies
    ... ... @@ -201,11 +208,12 @@ class Loader():
    201 208 
         #    rewritable (bool): Whether we should load in round trippable mode
    202 209 
         #    ticker (callable): A callback to report loaded filenames to the frontend
    203 210 
         #    fetch_subprojects (bool): Whether to fetch subprojects while loading
    211 
    +    #    yaml_cache (YamlCache): A yaml cache
    204 212 
         #
    205 213 
         # Returns:
    206 214 
         #    (LoadElement): A loaded LoadElement
    207 215 
         #
    208 
    -    def _load_file(self, filename, rewritable, ticker, fetch_subprojects):
    216 
    +    def _load_file(self, filename, rewritable, ticker, fetch_subprojects, yaml_cache=None):
    209 217
    210 218 
             # Silently ignore already loaded files
    211 219 
             if filename in self._elements:
    ... ... @@ -218,7 +226,8 @@ class Loader():
    218 226 
             # Load the data and process any conditional statements therein
    219 227 
             fullpath = os.path.join(self._basedir, filename)
    220 228 
             try:
    221 
    -            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable, project=self.project)
    229 
    +            node = _yaml.load(fullpath, shortname=filename, copy_tree=rewritable,
    230 
    +                              project=self.project, yaml_cache=yaml_cache)
    222 231 
             except LoadError as e:
    223 232 
                 if e.reason == LoadErrorReason.MISSING_FILE:
    224 233 
                     # If we can't find the file, try to suggest plausible
    ... ... @@ -261,13 +270,13 @@ class Loader():
    261 270 
             # Load all dependency files for the new LoadElement
    262 271 
             for dep in element.deps:
    263 272 
                 if dep.junction:
    264 
    -                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects)
    273 
    +                self._load_file(dep.junction, rewritable, ticker, fetch_subprojects, yaml_cache)
    265 274 
                     loader = self._get_loader(dep.junction, rewritable=rewritable, ticker=ticker,
    266 275 
                                               fetch_subprojects=fetch_subprojects)
    267 276 
                 else:
    268 277 
                     loader = self
    269 278
    270 
    -            dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects)
    279 
    +            dep_element = loader._load_file(dep.name, rewritable, ticker, fetch_subprojects, yaml_cache)
    271 280
    272 281 
                 if _yaml.node_get(dep_element.node, str, Symbol.KIND) == 'junction':
    273 282 
                     raise LoadError(LoadErrorReason.INVALID_DATA,

  • buildstream/_project.py
    ... ... @@ -19,6 +19,7 @@
    19 19 
     #        Tiago Gomes <tiago gomes codethink co uk>
    20 20
    21 21 
     import os
    22 
    +import hashlib
    22 23 
     from collections import Mapping, OrderedDict
    23 24 
     from pluginbase import PluginBase
    24 25 
     from . import utils

  • buildstream/_yaml.py
    ... ... @@ -183,20 +183,32 @@ class CompositeTypeError(CompositeError):
    183 183 
     #    shortname (str): The filename in shorthand for error reporting (or None)
    184 184 
     #    copy_tree (bool): Whether to make a copy, preserving the original toplevels
    185 185 
     #                      for later serialization
    186 
    +#    yaml_cache (YamlCache): A yaml cache to consult rather than parsing
    186 187 
     #
    187 188 
     # Returns (dict): A loaded copy of the YAML file with provenance information
    188 189 
     #
    189 190 
     # Raises: LoadError
    190 191 
     #
    191 
    -def load(filename, shortname=None, copy_tree=False, *, project=None):
    192 
    +def load(filename, shortname=None, copy_tree=False, *, project=None, yaml_cache=None):
    192 193 
         if not shortname:
    193 194 
             shortname = filename
    194 195
    195 196 
         file = ProvenanceFile(filename, shortname, project)
    196 197
    197 198 
         try:
    199 
    +        data = None
    198 200 
             with open(filename) as f:
    199 
    -            return load_data(f, file, copy_tree=copy_tree)
    201 
    +            contents = f.read()
    202 
    +        if yaml_cache:
    203 
    +            data, key = yaml_cache.get(project, filename, contents, copy_tree)
    204 
    +
    205 
    +        if not data:
    206 
    +            data = load_data(contents, file, copy_tree=copy_tree)
    207 
    +
    208 
    +        if yaml_cache:
    209 
    +            yaml_cache.put_from_key(project, filename, key, data)
    210 
    +
    211 
    +        return data
    200 212 
         except FileNotFoundError as e:
    201 213 
             raise LoadError(LoadErrorReason.MISSING_FILE,
    202 214 
                             "Could not find file at {}".format(filename)) from e

  • buildstream/_yamlcache.py
    1 
    +#
    2 
    +#  Copyright 2018 Bloomberg Finance LP
    3 
    +#
    4 
    +#  This program is free software; you can redistribute it and/or
    5 
    +#  modify it under the terms of the GNU Lesser General Public
    6 
    +#  License as published by the Free Software Foundation; either
    7 
    +#  version 2 of the License, or (at your option) any later version.
    8 
    +#
    9 
    +#  This library is distributed in the hope that it will be useful,
    10 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    11 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
    12 
    +#  Lesser General Public License for more details.
    13 
    +#
    14 
    +#  You should have received a copy of the GNU Lesser General Public
    15 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    16 
    +#
    17 
    +#  Authors:
    18 
    +#        Jonathan Maw <jonathan maw codethink co uk>
    19 
    +
    20 
    +import os
    21 
    +import pickle
    22 
    +import hashlib
    23 
    +import io
    24 
    +
    25 
    +import sys
    26 
    +
    27 
    +from contextlib import contextmanager
    28 
    +from collections import namedtuple
    29 
    +
    30 
    +from ._cachekey import generate_key
    31 
    +from ._context import Context
    32 
    +from . import utils, _yaml
    33 
    +
    34 
    +
    35 
    +YAML_CACHE_FILENAME = "yaml_cache.pickle"
    36 
    +
    37 
    +
    38 
    +# YamlCache()
    39 
    +#
    40 
    +# A cache that wraps around the loading of yaml in projects.
    41 
    +#
    42 
    +# The recommended way to use a YamlCache is:
    43 
    +#   with YamlCache.open(context) as yamlcache:
    44 
    +#     # Load all the yaml
    45 
    +#     ...
    46 
    +#
    47 
    +# Args:
    48 
    +#    context (Context): The invocation Context
    49 
    +#
    50 
    +class YamlCache():
    51 
    +
    52 
    +    def __init__(self, context):
    53 
    +        self._project_caches = {}
    54 
    +        self._context = context
    55 
    +
    56 
    +    ##################
    57 
    +    # Public Methods #
    58 
    +    ##################
    59 
    +
    60 
    +    # is_cached():
    61 
    +    #
    62 
    +    # Checks whether a file is cached.
    63 
    +    #
    64 
    +    # Args:
    65 
    +    #    project (Project): The project this file is in.
    66 
    +    #    filepath (str): The path to the file, *relative to the project's directory*.
    67 
    +    #
    68 
    +    # Returns:
    69 
    +    #    (bool): Whether the file is cached.
    70 
    +    def is_cached(self, project, filepath):
    71 
    +        cache_path = self._get_filepath(project, filepath)
    72 
    +        project_name = project.name if project else ""
    73 
    +        try:
    74 
    +            project_cache = self._project_caches[project_name]
    75 
    +            if cache_path in project_cache.elements:
    76 
    +                return True
    77 
    +        except KeyError:
    78 
    +            pass
    79 
    +        return False
    80 
    +
    81 
    +    # open():
    82 
    +    #
    83 
    +    # Return an instance of the YamlCache which writes to disk when it leaves scope.
    84 
    +    #
    85 
    +    # Args:
    86 
    +    #    context (Context): The context.
    87 
    +    #    cachefile (str): The path to the cache file.
    88 
    +    #
    89 
    +    # Returns:
    90 
    +    #    (YamlCache): A YamlCache.
    91 
    +    @staticmethod
    92 
    +    @contextmanager
    93 
    +    def open(context, cachefile):
    94 
    +        # Try to load from disk first
    95 
    +        cache = None
    96 
    +        if os.path.exists(cachefile):
    97 
    +            try:
    98 
    +                with open(cachefile, "rb") as f:
    99 
    +                    cache = BstUnpickler(f, context).load()
    100 
    +            except EOFError:
    101 
    +                # The file was empty
    102 
    +                pass
    103 
    +            except pickle.UnpicklingError as e:
    104 
    +                sys.stderr.write("Failed to load YamlCache, {}\n".format(e))
    105 
    +
    106 
    +        # Failed to load from disk, create a new one
    107 
    +        if not cache:
    108 
    +            cache = YamlCache(context)
    109 
    +
    110 
    +        yield cache
    111 
    +
    112 
    +        cache._write(cachefile)
    113 
    +
    114 
    +    # get_cache_file():
    115 
    +    #
    116 
    +    # Retrieves a path to the yaml cache file.
    117 
    +    #
    118 
    +    # Returns:
    119 
    +    #   (str): The path to the cache file
    120 
    +    @staticmethod
    121 
    +    def get_cache_file(top_dir):
    122 
    +        return os.path.join(top_dir, ".bst", YAML_CACHE_FILENAME)
    123 
    +
    124 
    +    # get():
    125 
    +    #
    126 
    +    # Gets a parsed file from the cache.
    127 
    +    #
    128 
    +    # Args:
    129 
    +    #    project (Project) or None: The project this file is in, if it exists.
    130 
    +    #    filepath (str): The absolute path to the file.
    131 
    +    #    contents (str): The contents of the file to be cached
    132 
    +    #    copy_tree (bool): Whether the data should make a copy when it's being generated
    133 
    +    #                      (i.e. exactly as when called in yaml)
    134 
    +    #
    135 
    +    # Returns:
    136 
    +    #    (decorated dict): The parsed yaml from the cache, or None if the file isn't in the cache.
    137 
    +    #    (str):            The key used to look up the parsed yaml in the cache
    138 
    +    def get(self, project, filepath, contents, copy_tree):
    139 
    +        key = self._calculate_key(contents, copy_tree)
    140 
    +        data = self._get(project, filepath, key)
    141 
    +        return data, key
    142 
    +
    143 
    +    # put():
    144 
    +    #
    145 
    +    # Puts a parsed file into the cache.
    146 
    +    #
    147 
    +    # Args:
    148 
    +    #    project (Project): The project this file is in.
    149 
    +    #    filepath (str): The path to the file.
    150 
    +    #    contents (str): The contents of the file that has been cached
    151 
    +    #    copy_tree (bool): Whether the data should make a copy when it's being generated
    152 
    +    #                      (i.e. exactly as when called in yaml)
    153 
    +    #    value (decorated dict): The data to put into the cache.
    154 
    +    def put(self, project, filepath, contents, copy_tree, value):
    155 
    +        key = self._calculate_key(contents, copy_tree)
    156 
    +        self.put_from_key(project, filepath, key, value)
    157 
    +
    158 
    +    # put_from_key():
    159 
    +    #
    160 
    +    # Put a parsed file into the cache when given a key.
    161 
    +    #
    162 
    +    # Args:
    163 
    +    #    project (Project): The project this file is in.
    164 
    +    #    filepath (str): The path to the file.
    165 
    +    #    key (str): The key to the file within the cache. Typically, this is the
    166 
    +    #               value of `calculate_key()` with the file's unparsed contents
    167 
    +    #               and any relevant metadata passed in.
    168 
    +    #    value (decorated dict): The data to put into the cache.
    169 
    +    def put_from_key(self, project, filepath, key, value):
    170 
    +        cache_path = self._get_filepath(project, filepath)
    171 
    +        project_name = project.name if project else ""
    172 
    +        try:
    173 
    +            project_cache = self._project_caches[project_name]
    174 
    +        except KeyError:
    175 
    +            project_cache = self._project_caches[project_name] = CachedProject({})
    176 
    +
    177 
    +        project_cache.elements[cache_path] = CachedYaml(key, value)
    178 
    +
    179 
    +    ###################
    180 
    +    # Private Methods #
    181 
    +    ###################
    182 
    +
    183 
    +    # Writes the yaml cache to the specified path.
    184 
    +    #
    185 
    +    # Args:
    186 
    +    #    path (str): The path to the cache file.
    187 
    +    def _write(self, path):
    188 
    +        parent_dir = os.path.dirname(path)
    189 
    +        os.makedirs(parent_dir, exist_ok=True)
    190 
    +        with open(path, "wb") as f:
    191 
    +            BstPickler(f).dump(self)
    192 
    +
    193 
    +    # _get_filepath():
    194 
    +    #
    195 
    +    # Returns a file path relative to a project if passed, or the original path if
    196 
    +    # the project is None
    197 
    +    #
    198 
    +    # Args:
    199 
    +    #    project (Project) or None: The project the filepath exists within
    200 
    +    #    full_path (str): The path that the returned path is based on
    201 
    +    #
    202 
    +    # Returns:
    203 
    +    #    (str): The path to the file, relative to a project if it exists
    204 
    +    def _get_filepath(self, project, full_path):
    205 
    +        if project:
    206 
    +            assert full_path.startswith(project.directory)
    207 
    +            filepath = os.path.relpath(full_path, project.directory)
    208 
    +        else:
    209 
    +            filepath = full_path
    210 
    +        return full_path
    211 
    +
    212 
    +    # _calculate_key():
    213 
    +    #
    214 
    +    # Calculates a key for putting into the cache.
    215 
    +    #
    216 
    +    # Args:
    217 
    +    #    (basic object)... : Any number of strictly-ordered basic objects
    218 
    +    #
    219 
    +    # Returns:
    220 
    +    #   (str): A key made out of every arg passed in
    221 
    +    @staticmethod
    222 
    +    def _calculate_key(*args):
    223 
    +        string = pickle.dumps(args)
    224 
    +        return hashlib.sha1(string).hexdigest()
    225 
    +
    226 
    +    # _get():
    227 
    +    #
    228 
    +    # Gets a parsed file from the cache when given a key.
    229 
    +    #
    230 
    +    # Args:
    231 
    +    #    project (Project): The project this file is in.
    232 
    +    #    filepath (str): The path to the file.
    233 
    +    #    key (str): The key to the file within the cache. Typically, this is the
    234 
    +    #               value of `calculate_key()` with the file's unparsed contents
    235 
    +    #               and any relevant metadata passed in.
    236 
    +    #
    237 
    +    # Returns:
    238 
    +    #    (decorated dict): The parsed yaml from the cache, or None if the file isn't in the cache.
    239 
    +    def _get(self, project, filepath, key):
    240 
    +        cache_path = self._get_filepath(project, filepath)
    241 
    +        project_name = project.name if project else ""
    242 
    +        try:
    243 
    +            project_cache = self._project_caches[project_name]
    244 
    +            try:
    245 
    +                cachedyaml = project_cache.elements[cache_path]
    246 
    +                if cachedyaml._key == key:
    247 
    +                    # We've unpickled the YamlCache, but not the specific file
    248 
    +                    if cachedyaml._contents is None:
    249 
    +                        cachedyaml._contents = BstUnpickler.loads(cachedyaml._pickled_contents, self._context)
    250 
    +                    return cachedyaml._contents
    251 
    +            except KeyError:
    252 
    +                pass
    253 
    +        except KeyError:
    254 
    +            pass
    255 
    +        return None
    256 
    +
    257 
    +
    258 
    +CachedProject = namedtuple('CachedProject', ['elements'])
    259 
    +
    260 
    +
    261 
    +class CachedYaml():
    262 
    +    def __init__(self, key, contents):
    263 
    +        self._key = key
    264 
    +        self.set_contents(contents)
    265 
    +
    266 
    +    # Sets the contents of the CachedYaml.
    267 
    +    #
    268 
    +    # Args:
    269 
    +    #    contents (provenanced dict): The contents to put in the cache.
    270 
    +    #
    271 
    +    def set_contents(self, contents):
    272 
    +        self._contents = contents
    273 
    +        self._pickled_contents = BstPickler.dumps(contents)
    274 
    +
    275 
    +    # Pickling helper method, prevents 'contents' from being serialised
    276 
    +    def __getstate__(self):
    277 
    +        data = self.__dict__.copy()
    278 
    +        data['_contents'] = None
    279 
    +        return data
    280 
    +
    281 
    +
    282 
    +# In _yaml.load, we have a ProvenanceFile that stores the project the file
    283 
    +# came from. Projects can't be pickled, but it's always going to be the same
    284 
    +# project between invocations (unless the entire project is moved but the
    285 
    +# file stayed in the same place)
    286 
    +class BstPickler(pickle.Pickler):
    287 
    +    def persistent_id(self, obj):
    288 
    +        if isinstance(obj, _yaml.ProvenanceFile):
    289 
    +            if obj.project:
    290 
    +                # ProvenanceFile's project object cannot be stored as it is.
    291 
    +                project_tag = obj.project.name
    292 
    +                # ProvenanceFile's filename must be stored relative to the
    293 
    +                # project, as the project dir may move.
    294 
    +                name = os.path.relpath(obj.name, obj.project.directory)
    295 
    +            else:
    296 
    +                project_tag = None
    297 
    +                name = obj.name
    298 
    +            return ("ProvenanceFile", name, obj.shortname, project_tag)
    299 
    +        elif isinstance(obj, Context):
    300 
    +            return ("Context",)
    301 
    +        else:
    302 
    +            return None
    303 
    +
    304 
    +    @staticmethod
    305 
    +    def dumps(obj):
    306 
    +        stream = io.BytesIO()
    307 
    +        BstPickler(stream).dump(obj)
    308 
    +        stream.seek(0)
    309 
    +        return stream.read()
    310 
    +
    311 
    +
    312 
    +class BstUnpickler(pickle.Unpickler):
    313 
    +    def __init__(self, file, context):
    314 
    +        super().__init__(file)
    315 
    +        self._context = context
    316 
    +
    317 
    +    def persistent_load(self, pid):
    318 
    +        if pid[0] == "ProvenanceFile":
    319 
    +            _, tagged_name, shortname, project_tag = pid
    320 
    +
    321 
    +            if project_tag is not None:
    322 
    +                for p in self._context.get_projects():
    323 
    +                    if project_tag == p.name:
    324 
    +                        project = p
    325 
    +                        break
    326 
    +
    327 
    +                name = os.path.join(project.directory, tagged_name)
    328 
    +
    329 
    +                if not project:
    330 
    +                    projects = [p.name for p in self._context.get_projects()]
    331 
    +                    raise pickle.UnpicklingError("No project with name {} found in {}"
    332 
    +                                                 .format(key_id, projects))
    333 
    +            else:
    334 
    +                project = None
    335 
    +                name = tagged_name
    336 
    +
    337 
    +            return _yaml.ProvenanceFile(name, shortname, project)
    338 
    +        elif pid[0] == "Context":
    339 
    +            return self._context
    340 
    +        else:
    341 
    +            raise pickle.UnpicklingError("Unsupported persistent object, {}".format(pid))
    342 
    +
    343 
    +    @staticmethod
    344 
    +    def loads(text, context):
    345 
    +        stream = io.BytesIO()
    346 
    +        stream.write(bytes(text))
    347 
    +        stream.seek(0)
    348 
    +        return BstUnpickler(stream, context).load()

  • buildstream/element.py
    ... ... @@ -86,7 +86,7 @@ from ._variables import Variables
    86 86 
     from ._versions import BST_CORE_ARTIFACT_VERSION
    87 87 
     from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, ErrorDomain
    88 88 
     from .utils import UtilError
    89 
    -from . import Plugin, Consistency
    89 
    +from . import Plugin, Consistency, Scope
    90 90 
     from . import SandboxFlags
    91 91 
     from . import utils
    92 92 
     from . import _cachekey
    ... ... @@ -96,11 +96,11 @@ from ._platform import Platform
    96 96 
     from .plugin import CoreWarnings
    97 97 
     from .sandbox._config import SandboxConfig
    98 98 
     from .sandbox._sandboxremote import SandboxRemote
    99 
    +from .types import _KeyStrength
    99 100
    100 101 
     from .storage.directory import Directory
    101 102 
     from .storage._filebaseddirectory import FileBasedDirectory
    102 103 
     from .storage.directory import VirtualDirectoryError
    103 
    -from .element_enums import _KeyStrength, Scope
    104 104
    105 105
    106 106 
     class ElementError(BstError):

  • buildstream/source.py
    ... ... @@ -145,35 +145,12 @@ import os
    145 145 
     from collections import Mapping
    146 146 
     from contextlib import contextmanager
    147 147
    148 
    -from . import Plugin
    148 
    +from . import Plugin, Consistency
    149 149 
     from . import _yaml, utils
    150 150 
     from ._exceptions import BstError, ImplError, ErrorDomain
    151 151 
     from ._projectrefs import ProjectRefStorage
    152 152
    153 153
    154 
    -class Consistency():
    155 
    -    INCONSISTENT = 0
    156 
    -    """Inconsistent
    157 
    -
    158 
    -    Inconsistent sources have no explicit reference set. They cannot
    159 
    -    produce a cache key, be fetched or staged. They can only be tracked.
    160 
    -    """
    161 
    -
    162 
    -    RESOLVED = 1
    163 
    -    """Resolved
    164 
    -
    165 
    -    Resolved sources have a reference and can produce a cache key and
    166 
    -    be fetched, however they cannot be staged.
    167 
    -    """
    168 
    -
    169 
    -    CACHED = 2
    170 
    -    """Cached
    171 
    -
    172 
    -    Cached sources have a reference which is present in the local
    173 
    -    source cache. Only cached sources can be staged.
    174 
    -    """
    175 
    -
    176 
    -
    177 154 
     class SourceError(BstError):
    178 155 
         """This exception should be raised by :class:`.Source` implementations
    179 156 
         to report errors to the user.

  • buildstream/element_enums.py → buildstream/types.py
    ... ... @@ -19,31 +19,19 @@
    19 19 
     #        Jim MacArthur <jim macarthur codethink co uk>
    20 20
    21 21 
     """
    22 
    -Element - Globally visible enumerations
    23 
    -=======================================
    22 
    +Foundation types
    23 
    +================
    24 24
    25 25 
     """
    26 26
    27 27 
     from enum import Enum
    28 28
    29 29
    30 
    -# _KeyStrength():
    31 
    -#
    32 
    -# Strength of cache key
    33 
    -#
    34 
    -class _KeyStrength(Enum):
    35 
    -
    36 
    -    # Includes strong cache keys of all build dependencies and their
    37 
    -    # runtime dependencies.
    38 
    -    STRONG = 1
    39 
    -
    40 
    -    # Includes names of direct build dependencies but does not include
    41 
    -    # cache keys of dependencies.
    42 
    -    WEAK = 2
    43 
    -
    44 
    -
    45 30 
     class Scope(Enum):
    46 
    -    """Types of scope for a given element"""
    31 
    +    """Defines the scope of dependencies to include for a given element
    32 
    +    when iterating over the dependency graph in APIs like
    33 
    +    :func:`Element.dependencies() <buildstream.element.Element.dependencies>`
    34 
    +    """
    47 35
    48 36 
         ALL = 1
    49 37 
         """All elements which the given element depends on, following
    ... ... @@ -59,3 +47,44 @@ class Scope(Enum):
    59 47 
         """All elements required for running the element. Including the element
    60 48 
         itself.
    61 49 
         """
    50 
    +
    51 
    +
    52 
    +class Consistency():
    53 
    +    """Defines the various consistency states of a :class:`.Source`.
    54 
    +    """
    55 
    +
    56 
    +    INCONSISTENT = 0
    57 
    +    """Inconsistent
    58 
    +
    59 
    +    Inconsistent sources have no explicit reference set. They cannot
    60 
    +    produce a cache key, be fetched or staged. They can only be tracked.
    61 
    +    """
    62 
    +
    63 
    +    RESOLVED = 1
    64 
    +    """Resolved
    65 
    +
    66 
    +    Resolved sources have a reference and can produce a cache key and
    67 
    +    be fetched, however they cannot be staged.
    68 
    +    """
    69 
    +
    70 
    +    CACHED = 2
    71 
    +    """Cached
    72 
    +
    73 
    +    Cached sources have a reference which is present in the local
    74 
    +    source cache. Only cached sources can be staged.
    75 
    +    """
    76 
    +
    77 
    +
    78 
    +# _KeyStrength():
    79 
    +#
    80 
    +# Strength of cache key
    81 
    +#
    82 
    +class _KeyStrength(Enum):
    83 
    +
    84 
    +    # Includes strong cache keys of all build dependencies and their
    85 
    +    # runtime dependencies.
    86 
    +    STRONG = 1
    87 
    +
    88 
    +    # Includes names of direct build dependencies but does not include
    89 
    +    # cache keys of dependencies.
    90 
    +    WEAK = 2

  • doc/source/additional_docker.rst
    1 
    +
    2 
    +.. _bst_and_docker:
    3 
    +
    4 
    +
    5 
    +BuildStream and Docker
    6 
    +======================
    7 
    +
    8 
    +BuildStream integrates with Docker in multiple ways. Here are some ways in
    9 
    +which these integrations work.
    10 
    +
    11 
    +Run BuildStream inside Docker
    12 
    +-----------------------------
    13 
    +
    14 
    +Refer to the :ref:`BuildStream inside Docker <docker>` documentation for
    15 
    +instructions on how to run BuildStream as a Docker container.
    16 
    +
    17 
    +Generate Docker images
    18 
    +----------------------
    19 
    +
    20 
    +The
    21 
    +`bst-docker-import script <https://gitlab.com/BuildStream/buildstream/blob/master/contrib/bst-docker-import>`_
    22 
    +can be used to generate a Docker image from built artifacts.
    23 
    +
    24 
    +You can download it and make it executable like this:
    25 
    +
    26 
    +.. code:: bash
    27 
    +
    28 
    +  mkdir -p ~/.local/bin
    29 
    +  curl --get https://gitlab.com/BuildStream/buildstream/raw/master/contrib/bst-docker-import > ~/.local/bin/bst-docker-import
    30 
    +  chmod +x ~/.local/bin/bst-docker-import
    31 
    +
    32 
    +Check if ``~/.local/bin`` appears in your PATH environment variable -- if it
    33 
    +doesn't, you should
    34 
    +`edit your ~/.profile so that it does <https://stackoverflow.com/questions/14637979/>`_.
    35 
    +
    36 
    +Once the script is available in your PATH and assuming you have Docker
    37 
    +installed, you can start using the ``bst-docker-import`` script. Here is a
    38 
    +minimal example to generate an image called ``bst-hello`` from an element
    39 
    +called ``hello.bst`` assuming it is already built:
    40 
    +
    41 
    +.. code:: bash
    42 
    +
    43 
    +  bst-docker-import -t bst-hello hello.bst
    44 
    +
    45 
    +This script can also be used if you are running BuildStream inside Docker. In
    46 
    +this case, you will need to supply the command that you are using to run
    47 
    +BuildStream using the ``-c`` option.  If you are using the
    48 
    +`bst-here wrapper script <https://gitlab.com/BuildStream/buildstream/blob/master/contrib/bst-here>`_,
    49 
    +you can achieve the same results as the above example like this:
    50 
    +
    51 
    +.. code:: bash
    52 
    +
    53 
    +  bst-docker-import -c bst-here -t bst-hello hello.bst

  • doc/source/core_additional.rst
    ... ... @@ -8,3 +8,4 @@ Additional writings
    8 8
    9 9 
        additional_cachekeys
    10 10 
        additional_sandboxing
    11 
    +   additional_docker

  • doc/source/core_framework.rst
    ... ... @@ -12,6 +12,7 @@ useful for working on BuildStream itself.
    12 12 
     .. toctree::
    13 13 
        :maxdepth: 1
    14 14
    15 
    +   buildstream.types
    15 16 
        buildstream.plugin
    16 17 
        buildstream.source
    17 18 
        buildstream.element

  • tests/frontend/yamlcache.py
    1 
    +import os
    2 
    +import pytest
    3 
    +import hashlib
    4 
    +import tempfile
    5 
    +from ruamel import yaml
    6 
    +
    7 
    +from tests.testutils import cli, generate_junction, create_element_size, create_repo
    8 
    +from buildstream import _yaml
    9 
    +from buildstream._yamlcache import YamlCache
    10 
    +from buildstream._project import Project
    11 
    +from buildstream._context import Context
    12 
    +from contextlib import contextmanager
    13 
    +
    14 
    +
    15 
    +def generate_project(tmpdir, ref_storage, with_junction, name="test"):
    16 
    +    if with_junction == 'junction':
    17 
    +        subproject_dir = generate_project(
    18 
    +            tmpdir, ref_storage,
    19 
    +            'no-junction', name='test-subproject'
    20 
    +        )
    21 
    +
    22 
    +    project_dir = os.path.join(tmpdir, name)
    23 
    +    os.makedirs(project_dir)
    24 
    +    # project.conf
    25 
    +    project_conf_path = os.path.join(project_dir, 'project.conf')
    26 
    +    elements_path = 'elements'
    27 
    +    project_conf = {
    28 
    +        'name': name,
    29 
    +        'element-path': elements_path,
    30 
    +        'ref-storage': ref_storage,
    31 
    +    }
    32 
    +    _yaml.dump(project_conf, project_conf_path)
    33 
    +
    34 
    +    # elements
    35 
    +    if with_junction == 'junction':
    36 
    +        junction_name = 'junction.bst'
    37 
    +        junction_dir = os.path.join(project_dir, elements_path)
    38 
    +        junction_path = os.path.join(project_dir, elements_path, junction_name)
    39 
    +        os.makedirs(junction_dir)
    40 
    +        generate_junction(tmpdir, subproject_dir, junction_path)
    41 
    +        element_depends = [{'junction': junction_name, 'filename': 'test.bst'}]
    42 
    +    else:
    43 
    +        element_depends = []
    44 
    +
    45 
    +    element_name = 'test.bst'
    46 
    +    create_element_size(element_name, project_dir, elements_path, element_depends, 1)
    47 
    +
    48 
    +    return project_dir
    49 
    +
    50 
    +
    51 
    +@contextmanager
    52 
    +def with_yamlcache(project_dir):
    53 
    +    context = Context()
    54 
    +    project = Project(project_dir, context)
    55 
    +    cache_file = YamlCache.get_cache_file(project_dir)
    56 
    +    with YamlCache.open(context, cache_file) as yamlcache:
    57 
    +        yield yamlcache, project
    58 
    +
    59 
    +
    60 
    +def yamlcache_key(yamlcache, in_file, copy_tree=False):
    61 
    +    with open(in_file) as f:
    62 
    +        key = yamlcache._calculate_key(f.read(), copy_tree)
    63 
    +    return key
    64 
    +
    65 
    +
    66 
    +def modified_file(input_file, tmpdir):
    67 
    +    with open(input_file) as f:
    68 
    +        data = f.read()
    69 
    +    assert 'variables' not in data
    70 
    +    data += '\nvariables: {modified: True}\n'
    71 
    +    _, temppath = tempfile.mkstemp(dir=tmpdir, text=True)
    72 
    +    with open(temppath, 'w') as f:
    73 
    +        f.write(data)
    74 
    +
    75 
    +    return temppath
    76 
    +
    77 
    +
    78 
    +@pytest.mark.parametrize('ref_storage', ['inline', 'project.refs'])
    79 
    +@pytest.mark.parametrize('with_junction', ['no-junction', 'junction'])
    80 
    +@pytest.mark.parametrize('move_project', ['move', 'no-move'])
    81 
    +def test_yamlcache_used(cli, tmpdir, ref_storage, with_junction, move_project):
    82 
    +    # Generate the project
    83 
    +    project = generate_project(str(tmpdir), ref_storage, with_junction)
    84 
    +    if with_junction == 'junction':
    85 
    +        result = cli.run(project=project, args=['fetch', '--track', 'junction.bst'])
    86 
    +        result.assert_success()
    87 
    +
    88 
    +    # bst show to put it in the cache
    89 
    +    result = cli.run(project=project, args=['show', 'test.bst'])
    90 
    +    result.assert_success()
    91 
    +
    92 
    +    element_path = os.path.join(project, 'elements', 'test.bst')
    93 
    +    with with_yamlcache(project) as (yc, prj):
    94 
    +        # Check that it's in the cache
    95 
    +        assert yc.is_cached(prj, element_path)
    96 
    +
    97 
    +        # *Absolutely* horrible cache corruption to check it's being used
    98 
    +        # Modifying the data from the cache is fraught with danger,
    99 
    +        # so instead I'll load a modified version of the original file
    100 
    +        temppath = modified_file(element_path, str(tmpdir))
    101 
    +        contents = _yaml.load(temppath, copy_tree=False, project=prj)
    102 
    +        key = yamlcache_key(yc, element_path)
    103 
    +        yc.put_from_key(prj, element_path, key, contents)
    104 
    +
    105 
    +    # Show that a variable has been added
    106 
    +    result = cli.run(project=project, args=['show', '--format', '%{vars}', 'test.bst'])
    107 
    +    result.assert_success()
    108 
    +    data = yaml.safe_load(result.output)
    109 
    +    assert 'modified' in data
    110 
    +    assert data['modified'] == 'True'
    111 
    +
    112 
    +
    113 
    +@pytest.mark.parametrize('ref_storage', ['inline', 'project.refs'])
    114 
    +@pytest.mark.parametrize('with_junction', ['junction', 'no-junction'])
    115 
    +@pytest.mark.parametrize('move_project', ['move', 'no-move'])
    116 
    +def test_yamlcache_changed_file(cli, ref_storage, with_junction, move_project):
    117 
    +    # i.e. a file is cached, the file is changed, loading the file (with cache) returns new data
    118 
    +    # inline and junction can only be changed by opening a workspace
    119 
    +    pass

  • tests/yaml/yaml.py
    1 1 
     import os
    2 2 
     import pytest
    3 
    +import tempfile
    3 4 
     from collections import Mapping
    4 5
    5 6 
     from buildstream import _yaml
    6 7 
     from buildstream._exceptions import LoadError, LoadErrorReason
    8 
    +from buildstream._context import Context
    9 
    +from buildstream._yamlcache import YamlCache
    7 10
    8 11 
     DATA_DIR = os.path.join(
    9 12 
         os.path.dirname(os.path.realpath(__file__)),
    ... ... @@ -150,6 +153,21 @@ def test_composite_preserve_originals(datafiles):
    150 153 
         assert(_yaml.node_get(orig_extra, str, 'old') == 'new')
    151 154
    152 155
    156 
    +def load_yaml_file(filename, *, cache_path, shortname=None, from_cache='raw'):
    157 
    +
    158 
    +    _, temppath = tempfile.mkstemp(dir=os.path.join(cache_path.dirname, cache_path.basename), text=True)
    159 
    +    context = Context()
    160 
    +
    161 
    +    with YamlCache.open(context, temppath) as yc:
    162 
    +        if from_cache == 'raw':
    163 
    +            return _yaml.load(filename, shortname)
    164 
    +        elif from_cache == 'cached':
    165 
    +            _yaml.load(filename, shortname, yaml_cache=yc)
    166 
    +            return _yaml.load(filename, shortname, yaml_cache=yc)
    167 
    +        else:
    168 
    +            assert False
    169 
    +
    170 
    +
    153 171 
     # Tests for list composition
    154 172 
     #
    155 173 
     # Each test composits a filename on top of basics.yaml, and tests
    ... ... @@ -165,6 +183,7 @@ def test_composite_preserve_originals(datafiles):
    165 183 
     #    prov_col: The expected provenance column of "mood"
    166 184 
     #
    167 185 
     @pytest.mark.datafiles(os.path.join(DATA_DIR))
    186 
    +@pytest.mark.parametrize('caching', [('raw'), ('cached')])
    168 187 
     @pytest.mark.parametrize("filename,index,length,mood,prov_file,prov_line,prov_col", [
    169 188
    170 189 
         # Test results of compositing with the (<) prepend directive
    ... ... @@ -195,14 +214,15 @@ def test_composite_preserve_originals(datafiles):
    195 214 
         ('implicitoverwrite.yaml', 0, 2, 'overwrite1', 'implicitoverwrite.yaml', 4, 8),
    196 215 
         ('implicitoverwrite.yaml', 1, 2, 'overwrite2', 'implicitoverwrite.yaml', 6, 8),
    197 216 
     ])
    198 
    -def test_list_composition(datafiles, filename,
    217 
    +def test_list_composition(datafiles, filename, tmpdir,
    199 218 
                               index, length, mood,
    200 
    -                          prov_file, prov_line, prov_col):
    201 
    -    base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    202 
    -    overlay = os.path.join(datafiles.dirname, datafiles.basename, filename)
    219 
    +                          prov_file, prov_line, prov_col, caching):
    220 
    +    base_file = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    221 
    +    overlay_file = os.path.join(datafiles.dirname, datafiles.basename, filename)
    222 
    +
    223 
    +    base = load_yaml_file(base_file, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    224 
    +    overlay = load_yaml_file(overlay_file, cache_path=tmpdir, shortname=filename, from_cache=caching)
    203 225
    204 
    -    base = _yaml.load(base, shortname='basics.yaml')
    205 
    -    overlay = _yaml.load(overlay, shortname=filename)
    206 226 
         _yaml.composite_dict(base, overlay)
    207 227
    208 228 
         children = _yaml.node_get(base, list, 'children')
    ... ... @@ -254,6 +274,7 @@ def test_list_deletion(datafiles):
    254 274 
     #    prov_col: The expected provenance column of "mood"
    255 275 
     #
    256 276 
     @pytest.mark.datafiles(os.path.join(DATA_DIR))
    277 
    +@pytest.mark.parametrize('caching', [('raw'), ('cached')])
    257 278 
     @pytest.mark.parametrize("filename1,filename2,index,length,mood,prov_file,prov_line,prov_col", [
    258 279
    259 280 
         # Test results of compositing literal list with (>) and then (<)
    ... ... @@ -310,9 +331,9 @@ def test_list_deletion(datafiles):
    310 331 
         ('listoverwrite.yaml', 'listprepend.yaml', 2, 4, 'overwrite1', 'listoverwrite.yaml', 5, 10),
    311 332 
         ('listoverwrite.yaml', 'listprepend.yaml', 3, 4, 'overwrite2', 'listoverwrite.yaml', 7, 10),
    312 333 
     ])
    313 
    -def test_list_composition_twice(datafiles, filename1, filename2,
    334 
    +def test_list_composition_twice(datafiles, tmpdir, filename1, filename2,
    314 335 
                                     index, length, mood,
    315 
    -                                prov_file, prov_line, prov_col):
    336 
    +                                prov_file, prov_line, prov_col, caching):
    316 337 
         file_base = os.path.join(datafiles.dirname, datafiles.basename, 'basics.yaml')
    317 338 
         file1 = os.path.join(datafiles.dirname, datafiles.basename, filename1)
    318 339 
         file2 = os.path.join(datafiles.dirname, datafiles.basename, filename2)
    ... ... @@ -320,9 +341,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
    320 341 
         #####################
    321 342 
         # Round 1 - Fight !
    322 343 
         #####################
    323 
    -    base = _yaml.load(file_base, shortname='basics.yaml')
    324 
    -    overlay1 = _yaml.load(file1, shortname=filename1)
    325 
    -    overlay2 = _yaml.load(file2, shortname=filename2)
    344 
    +    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    345 
    +    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    346 
    +    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
    326 347
    327 348 
         _yaml.composite_dict(base, overlay1)
    328 349 
         _yaml.composite_dict(base, overlay2)
    ... ... @@ -337,9 +358,9 @@ def test_list_composition_twice(datafiles, filename1, filename2,
    337 358 
         #####################
    338 359 
         # Round 2 - Fight !
    339 360 
         #####################
    340 
    -    base = _yaml.load(file_base, shortname='basics.yaml')
    341 
    -    overlay1 = _yaml.load(file1, shortname=filename1)
    342 
    -    overlay2 = _yaml.load(file2, shortname=filename2)
    361 
    +    base = load_yaml_file(file_base, cache_path=tmpdir, shortname='basics.yaml', from_cache=caching)
    362 
    +    overlay1 = load_yaml_file(file1, cache_path=tmpdir, shortname=filename1, from_cache=caching)
    363 
    +    overlay2 = load_yaml_file(file2, cache_path=tmpdir, shortname=filename2, from_cache=caching)
    343 364
    344 365 
         _yaml.composite_dict(overlay1, overlay2)
    345 366 
         _yaml.composite_dict(base, overlay1)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]