GNOME.org
 

[Notes] [Git][BuildStream/buildstream][are_you_sure2] 7 commits: Variables: Rework how expansion strings work

Title: GitLab

Angelos Evripiotis pushed to branch are_you_sure2 at BuildStream / buildstream

Commits:

14 changed files:

Changes:

  • buildstream/_context.py
    ... ... @@ -124,10 +124,6 @@ class Context():
    124 124 
             # Whether or not to cache build trees on artifact creation
    125 125 
             self.cache_buildtrees = None
    126 126
    127 
    -        # Boolean, whether we double-check with the user that they meant to
    128 
    -        # close the workspace when they're using it to access the project.
    129 
    -        self.prompt_workspace_close_project_inaccessible = None
    130 
    -
    131 127 
             # Whether elements must be rebuilt when their dependencies have changed
    132 128 
             self._strict_build_plan = None
    133 129
    ... ... @@ -248,22 +244,6 @@ class Context():
    248 244 
             self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers')
    249 245 
             self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries')
    250 246
    251 
    -        # Load prompt preferences
    252 
    -        #
    253 
    -        # We convert string options to booleans here, so we can be both user
    254 
    -        # and coder-friendly. The string options are worded to match the
    255 
    -        # responses the user would give at the cli, for least surprise. The
    256 
    -        # booleans are converted here because it's easiest to eyeball that the
    257 
    -        # strings are right.
    258 
    -        #
    259 
    -        prompt = _yaml.node_get(
    260 
    -            defaults, Mapping, 'prompt')
    261 
    -        _yaml.node_validate(prompt, [
    262 
    -            'really-workspace-close-project-inaccessible',
    263 
    -        ])
    264 
    -        self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
    265 
    -            prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
    266 
    -
    267 247 
             # Load per-projects overrides
    268 248 
             self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})
    269 249

  • buildstream/_frontend/cli.py
    ... ... @@ -814,6 +814,8 @@ def workspace_open(app, no_checkout, force, track_, directory, elements):
    814 814 
     def workspace_close(app, remove_dir, all_, elements):
    815 815 
         """Close a workspace"""
    816 816
    817 
    +    removed_required_element = False
    818 
    +
    817 819 
         with app.initialized():
    818 820 
             if not (all_ or elements):
    819 821 
                 # NOTE: I may need to revisit this when implementing multiple projects
    ... ... @@ -840,18 +842,20 @@ def workspace_close(app, remove_dir, all_, elements):
    840 842 
             for element_name in elements:
    841 843 
                 if not app.stream.workspace_exists(element_name):
    842 844 
                     nonexisting.append(element_name)
    843 
    -            if (app.stream.workspace_is_required(element_name) and app.interactive and
    844 
    -                    app.context.prompt_workspace_close_project_inaccessible):
    845 
    -                click.echo("Removing '{}' will prevent you from running "
    846 
    -                           "BuildStream commands from the current directory".format(element_name))
    847 
    -                if not click.confirm('Are you sure you want to close this workspace?'):
    848 
    -                    click.echo('Aborting', err=True)
    849 
    -                    sys.exit(-1)
    850 845 
             if nonexisting:
    851 846 
                 raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
    852 847
    853 848 
             for element_name in elements:
    854 849 
                 app.stream.workspace_close(element_name, remove_dir=remove_dir)
    850 
    +            if app.stream.workspace_is_required(element_name):
    851 
    +                removed_required_element = True
    852 
    +
    853 
    +    # This message is echo'd last, as it's most relevant to the next
    854 
    +    # thing the user will type.
    855 
    +    if removed_required_element:
    856 
    +        click.echo(
    857 
    +            "Removed '{}', therefore you can no longer run BuildStream "
    858 
    +            "commands from the current directory.".format(element_name), err=True)
    855 859
    856 860
    857 861 
     ##################################################################

  • buildstream/_frontend/widget.py
    ... ... @@ -398,7 +398,7 @@ class LogLine(Widget):
    398 398
    399 399 
                 # Variables
    400 400 
                 if "%{vars" in format_:
    401 
    -                variables = _yaml.node_sanitize(element._Element__variables.variables)
    401 
    +                variables = _yaml.node_sanitize(element._Element__variables.flat)
    402 402 
                     line = p.fmt_subst(
    403 403 
                         line, 'vars',
    404 404 
                         yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))

  • buildstream/_loader/loader.py
    ... ... @@ -284,17 +284,17 @@ class Loader():
    284 284 
         def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
    285 285
    286 286 
             if check_elements is None:
    287 
    -            check_elements = {}
    287 
    +            check_elements = set()
    288 288 
             if validated is None:
    289 
    -            validated = {}
    289 
    +            validated = set()
    290 290 
             if sequence is None:
    291 291 
                 sequence = []
    292 292
    293 293 
             # Skip already validated branches
    294 
    -        if validated.get(element) is not None:
    294 
    +        if element in validated:
    295 295 
                 return
    296 296
    297 
    -        if check_elements.get(element) is not None:
    297 
    +        if element in check_elements:
    298 298 
                 # Create `chain`, the loop of element dependencies from this
    299 299 
                 # element back to itself, by trimming everything before this
    300 300 
                 # element from the sequence under consideration.
    ... ... @@ -306,15 +306,15 @@ class Loader():
    306 306 
                                 .format(element.full_name, " -> ".join(chain)))
    307 307
    308 308 
             # Push / Check each dependency / Pop
    309 
    -        check_elements[element] = True
    309 
    +        check_elements.add(element)
    310 310 
             sequence.append(element.full_name)
    311 311 
             for dep in element.dependencies:
    312 312 
                 dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
    313 
    -        del check_elements[element]
    313 
    +        check_elements.remove(element)
    314 314 
             sequence.pop()
    315 315
    316 316 
             # Eliminate duplicate paths
    317 
    -        validated[element] = True
    317 
    +        validated.add(element)
    318 318
    319 319 
         # _sort_dependencies():
    320 320 
         #

  • buildstream/_profile.py
    ... ... @@ -26,7 +26,7 @@ import datetime
    26 26 
     import time
    27 27
    28 28 
     # Track what profile topics are active
    29 
    -active_topics = {}
    29 
    +active_topics = set()
    30 30 
     active_profiles = {}
    31 31 
     initialized = False
    32 32
    ... ... @@ -144,14 +144,10 @@ def profile_init():
    144 144 
             if setting:
    145 145 
                 topics = setting.split(':')
    146 146 
                 for topic in topics:
    147 
    -                active_topics[topic] = True
    147 
    +                active_topics.add(topic)
    148 148 
             initialized = True
    149 149
    150 150
    151 151 
     def profile_enabled(topic):
    152 152 
         profile_init()
    153 
    -    if active_topics.get(topic):
    154 
    -        return True
    155 
    -    if active_topics.get(Topics.ALL):
    156 
    -        return True
    157 
    -    return False
    153 
    +    return topic in active_topics or Topics.ALL in active_topics

  • buildstream/_variables.py
    1 1 
     #
    2 2 
     #  Copyright (C) 2016 Codethink Limited
    3 
    +#  Copyright (C) 2019 Bloomberg L.P.
    3 4 
     #
    4 5 
     #  This program is free software; you can redistribute it and/or
    5 6 
     #  modify it under the terms of the GNU Lesser General Public
    ... ... @@ -16,15 +17,37 @@
    16 17 
     #
    17 18 
     #  Authors:
    18 19 
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    20 
    +#        Daniel Silverstone <daniel silverstone codethink co uk>
    19 21
    20 22 
     import re
    23 
    +import sys
    21 24
    22 25 
     from ._exceptions import LoadError, LoadErrorReason
    23 26 
     from . import _yaml
    24 27
    25 28 
     # Variables are allowed to have dashes here
    26 29 
     #
    27 
    -_VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
    30 
    +PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
    31 
    +
    32 
    +
    33 
    +# Throughout this code you will see variables named things like `expstr`.
    34 
    +# These hold data structures called "expansion strings" and are the parsed
    35 
    +# form of the strings which are the input to this subsystem.  Strings
    36 
    +# such as "Hello %{name}, how are you?" are parsed into the form:
    37 
    +# (3, ["Hello ", "name", ", how are you?"])
    38 
    +# i.e. a tuple of an integer and a list, where the integer is the cached
    39 
    +# length of the list, and the list consists of one or more strings.
    40 
    +# Strings in even indices of the list (0, 2, 4, etc) are constants which
    41 
    +# are copied into the output of the expansion algorithm.  Strings in the
    42 
    +# odd indices (1, 3, 5, etc) are the names of further expansions to make.
    43 
    +# In the example above, first "Hello " is copied, then "name" is expanded
    44 
    +# and so must be another named expansion string passed in to the constructor
    45 
    +# of the Variables class, and whatever is yielded from the expansion of "name"
    46 
    +# is added to the concatenation for the result.  Finally ", how are you?" is
    47 
    +# copied in and the whole lot concatenated for return.
    48 
    +#
    49 
    +# To see how strings are parsed, see `_parse_expstr()` after the class, and
    50 
    +# to see how expansion strings are expanded, see `_expand_expstr()` after that.
    28 51
    29 52
    30 53 
     # The Variables helper object will resolve the variable references in
    ... ... @@ -38,14 +61,15 @@ _VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
    38 61 
     #     node (dict): A node loaded and composited with yaml tools
    39 62 
     #
    40 63 
     # Raises:
    41 
    -#     LoadError, if unresolved variables occur.
    64 
    +#     LoadError, if unresolved variables, or cycles in resolution, occur.
    42 65 
     #
    43 66 
     class Variables():
    44 67
    45 68 
         def __init__(self, node):
    46 69
    47 70 
             self.original = node
    48 
    -        self.variables = self._resolve(node)
    71 
    +        self._expstr_map = self._resolve(node)
    72 
    +        self.flat = self._flatten()
    49 73
    50 74 
         # subst():
    51 75 
         #
    ... ... @@ -61,139 +85,167 @@ class Variables():
    61 85 
         #    LoadError, if the string contains unresolved variable references.
    62 86 
         #
    63 87 
         def subst(self, string):
    64 
    -        substitute, unmatched, _ = self._subst(string, self.variables)
    65 
    -        unmatched = list(set(unmatched))
    66 
    -        if unmatched:
    67 
    -            if len(unmatched) == 1:
    68 
    -                message = "Unresolved variable '{var}'".format(var=unmatched[0])
    69 
    -            else:
    70 
    -                message = "Unresolved variables: "
    71 
    -                for unmatch in unmatched:
    72 
    -                    if unmatched.index(unmatch) > 0:
    73 
    -                        message += ', '
    74 
    -                    message += unmatch
    75 
    -
    76 
    -            raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
    77 
    -
    78 
    -        return substitute
    79 
    -
    80 
    -    def _subst(self, string, variables):
    81 
    -
    82 
    -        def subst_callback(match):
    83 
    -            nonlocal variables
    84 
    -            nonlocal unmatched
    85 
    -            nonlocal matched
    86 
    -
    87 
    -            token = match.group(0)
    88 
    -            varname = match.group(1)
    89 
    -
    90 
    -            value = _yaml.node_get(variables, str, varname, default_value=None)
    91 
    -            if value is not None:
    92 
    -                # We have to check if the inner string has variables
    93 
    -                # and return unmatches for those
    94 
    -                unmatched += re.findall(_VARIABLE_MATCH, value)
    95 
    -                matched += [varname]
    96 
    -            else:
    97 
    -                # Return unmodified token
    98 
    -                unmatched += [varname]
    99 
    -                value = token
    100 
    -
    101 
    -            return value
    102 
    -
    103 
    -        matched = []
    104 
    -        unmatched = []
    105 
    -        replacement = re.sub(_VARIABLE_MATCH, subst_callback, string)
    106 
    -
    107 
    -        return (replacement, unmatched, matched)
    88 
    +        expstr = _parse_expstr(string)
    89 
    +
    90 
    +        try:
    91 
    +            return _expand_expstr(self._expstr_map, expstr)
    92 
    +        except KeyError:
    93 
    +            unmatched = []
    94 
    +
    95 
    +            # Look for any unmatched variable names in the expansion string
    96 
    +            for var in expstr[1][1::2]:
    97 
    +                if var not in self._expstr_map:
    98 
    +                    unmatched.append(var)
    99 
    +
    100 
    +            if unmatched:
    101 
    +                message = "Unresolved variable{}: {}".format(
    102 
    +                    "s" if len(unmatched) > 1 else "",
    103 
    +                    ", ".join(unmatched)
    104 
    +                )
    105 
    +
    106 
    +                raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
    107 
    +            # Otherwise, re-raise the KeyError since it clearly came from some
    108 
    +            # other unknowable cause.
    109 
    +            raise
    108 110
    109 111 
         # Variable resolving code
    110 112 
         #
    111 
    -    # Here we substitute variables for values (resolve variables) repeatedly
    112 
    -    # in a dictionary, each time creating a new dictionary until there is no
    113 
    -    # more unresolved variables to resolve, or, until resolving further no
    114 
    -    # longer resolves anything, in which case we throw an exception.
    113 
    +    # Here we resolve all of our inputs into a dictionary, ready for use
    114 
    +    # in subst()
    115 115 
         def _resolve(self, node):
    116 
    -        variables = node
    117 
    -
    118 116 
             # Special case, if notparallel is specified in the variables for this
    119 117 
             # element, then override max-jobs to be 1.
    120 118 
             # Initialize it as a string as all variables are processed as strings.
    121 119 
             #
    122 
    -        if _yaml.node_get(variables, bool, 'notparallel', default_value=False):
    123 
    -            variables['max-jobs'] = str(1)
    124 
    -
    125 
    -        # Resolve the dictionary once, reporting the new dictionary with things
    126 
    -        # substituted in it, and reporting unmatched tokens.
    127 
    -        #
    128 
    -        def resolve_one(variables):
    129 
    -            unmatched = []
    130 
    -            resolved = {}
    131 
    -
    132 
    -            for key, value in _yaml.node_items(variables):
    133 
    -
    134 
    -                # Ensure stringness of the value before substitution
    135 
    -                value = _yaml.node_get(variables, str, key)
    136 
    -
    137 
    -                resolved_var, item_unmatched, matched = self._subst(value, variables)
    138 
    -
    139 
    -                if _wrap_variable(key) in resolved_var:
    140 
    -                    referenced_through = find_recursive_variable(key, matched, variables)
    120 
    +        if _yaml.node_get(node, bool, 'notparallel', default_value=False):
    121 
    +            node['max-jobs'] = str(1)
    122 
    +
    123 
    +        ret = {}
    124 
    +        for key, value in _yaml.node_items(node):
    125 
    +            value = _yaml.node_get(node, str, key)
    126 
    +            ret[sys.intern(key)] = _parse_expstr(value)
    127 
    +        return ret
    128 
    +
    129 
    +    def _check_for_missing(self):
    130 
    +        # First the check for anything unresolvable
    131 
    +        summary = []
    132 
    +        for key, expstr in self._expstr_map.items():
    133 
    +            for var in expstr[1][1::2]:
    134 
    +                if var not in self._expstr_map:
    135 
    +                    line = "  unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
    136 
    +                    provenance = _yaml.node_get_provenance(self.original, key)
    137 
    +                    summary.append(line.format(unmatched=var, variable=key, provenance=provenance))
    138 
    +        if summary:
    139 
    +            raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
    140 
    +                            "Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
    141 
    +
    142 
    +    def _check_for_cycles(self):
    143 
    +        # And now the cycle checks
    144 
    +        def cycle_check(expstr, visited, cleared):
    145 
    +            for var in expstr[1][1::2]:
    146 
    +                if var in cleared:
    147 
    +                    continue
    148 
    +                if var in visited:
    141 149 
                         raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
    142 
    -                                    "{}: ".format(_yaml.node_get_provenance(variables, key)) +
    150 
    +                                    "{}: ".format(_yaml.node_get_provenance(self.original, var)) +
    143 151 
                                         ("Variable '{}' expands to contain a reference to itself. " +
    144 
    -                                     "Perhaps '{}' contains '{}").format(key, referenced_through, _wrap_variable(key)))
    145 
    -
    146 
    -                resolved[key] = resolved_var
    147 
    -                unmatched += item_unmatched
    148 
    -
    149 
    -            # Carry over provenance
    150 
    -            resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
    151 
    -            return (resolved, unmatched)
    152 
    -
    153 
    -        # Resolve it until it's resolved or broken
    154 
    -        #
    155 
    -        resolved = variables
    156 
    -        unmatched = ['dummy']
    157 
    -        last_unmatched = ['dummy']
    158 
    -        while unmatched:
    159 
    -            resolved, unmatched = resolve_one(resolved)
    160 
    -
    161 
    -            # Lists of strings can be compared like this
    162 
    -            if unmatched == last_unmatched:
    163 
    -                # We've got the same result twice without matching everything,
    164 
    -                # something is undeclared or cyclic, compose a summary.
    165 
    -                #
    166 
    -                summary = ''
    167 
    -                for unmatch in set(unmatched):
    168 
    -                    for var, provenance in self._find_references(unmatch):
    169 
    -                        line = "  unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}\n"
    170 
    -                        summary += line.format(unmatched=unmatch, variable=var, provenance=provenance)
    171 
    -
    172 
    -                raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
    173 
    -                                "Failed to resolve one or more variable:\n{}".format(summary))
    174 
    -
    175 
    -            last_unmatched = unmatched
    176 
    -
    177 
    -        return resolved
    178 
    -
    179 
    -    # Helper function to fetch information about the node referring to a variable
    152 
    +                                     "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
    153 
    +                visited.append(var)
    154 
    +                cycle_check(self._expstr_map[var], visited, cleared)
    155 
    +                visited.pop()
    156 
    +                cleared.add(var)
    157 
    +
    158 
    +        cleared = set()
    159 
    +        for key, expstr in self._expstr_map.items():
    160 
    +            if key not in cleared:
    161 
    +                cycle_check(expstr, [key], cleared)
    162 
    +
    163 
    +    # _flatten():
    180 164 
         #
    181 
    -    def _find_references(self, varname):
    182 
    -        fullname = _wrap_variable(varname)
    183 
    -        for key, value in _yaml.node_items(self.original):
    184 
    -            if fullname in value:
    185 
    -                provenance = _yaml.node_get_provenance(self.original, key)
    186 
    -                yield (key, provenance)
    187 
    -
    188 
    -
    189 
    -def find_recursive_variable(variable, matched_variables, all_vars):
    190 
    -    matched_values = (_yaml.node_get(all_vars, str, key) for key in matched_variables)
    191 
    -    for key, value in zip(matched_variables, matched_values):
    192 
    -        if _wrap_variable(variable) in value:
    193 
    -            return key
    194 
    -    # We failed to find a recursive variable
    195 
    -    return None
    196 
    -
    197 
    -
    198 
    -def _wrap_variable(var):
    199 
    -    return "%{" + var + "}"
    165 
    +    # Turn our dictionary of expansion strings into a flattened dict
    166 
    +    # so that we can run expansions faster in the future
    167 
    +    #
    168 
    +    # Raises:
    169 
    +    #    LoadError, if the string contains unresolved variable references or
    170 
    +    #               if cycles are detected in the variable references
    171 
    +    #
    172 
    +    def _flatten(self):
    173 
    +        flat = {}
    174 
    +        try:
    175 
    +            for key, expstr in self._expstr_map.items():
    176 
    +                if expstr[0] > 1:
    177 
    +                    expstr = (1, [sys.intern(_expand_expstr(self._expstr_map, expstr))])
    178 
    +                    self._expstr_map[key] = expstr
    179 
    +                flat[key] = expstr[1][0]
    180 
    +        except KeyError:
    181 
    +            self._check_for_missing()
    182 
    +            raise
    183 
    +        except RecursionError:
    184 
    +            self._check_for_cycles()
    185 
    +            raise
    186 
    +        return flat
    187 
    +
    188 
    +
    189 
    +# Cache for the parsed expansion strings.  While this is nominally
    190 
    +# something which might "waste" memory, in reality each of these
    191 
    +# will live as long as the element which uses it, which is the
    192 
    +# vast majority of the memory usage across the execution of BuildStream.
    193 
    +PARSE_CACHE = {
    194 
    +    # Prime the cache with the empty string since otherwise that can
    195 
    +    # cause issues with the parser, complications to which cause slowdown
    196 
    +    "": (1, [""]),
    197 
    +}
    198 
    +
    199 
    +
    200 
    +# Helper to parse a string into an expansion string tuple, caching
    201 
    +# the results so that future parse requests don't need to think about
    202 
    +# the string
    203 
    +def _parse_expstr(instr):
    204 
    +    try:
    205 
    +        return PARSE_CACHE[instr]
    206 
    +    except KeyError:
    207 
    +        # This use of the regex turns a string like "foo %{bar} baz" into
    208 
    +        # a list ["foo ", "bar", " baz"]
    209 
    +        splits = PARSE_EXPANSION.split(instr)
    210 
    +        # If an expansion ends the string, we get an empty string on the end
    211 
    +        # which we can optimise away, making the expansion routines not need
    212 
    +        # a test for this.
    213 
    +        if splits[-1] == '':
    214 
    +            splits = splits[:-1]
    215 
    +        # Cache an interned copy of this.  We intern it to try and reduce the
    216 
    +        # memory impact of the cache.  It seems odd to cache the list length
    217 
    +        # but this is measurably cheaper than calculating it each time during
    218 
    +        # string expansion.
    219 
    +        PARSE_CACHE[instr] = (len(splits), [sys.intern(s) for s in splits])
    220 
    +        return PARSE_CACHE[instr]
    221 
    +
    222 
    +
    223 
    +# Helper to expand a given top level expansion string tuple in the context
    224 
    +# of the given dictionary of expansion strings.
    225 
    +#
    226 
    +# Note: Will raise KeyError if any expansion is missing
    227 
    +def _expand_expstr(content, topvalue):
    228 
    +    # Short-circuit constant strings
    229 
    +    if topvalue[0] == 1:
    230 
    +        return topvalue[1][0]
    231 
    +
    232 
    +    # Short-circuit strings which are entirely an expansion of another variable
    233 
    +    # e.g. "%{another}"
    234 
    +    if topvalue[0] == 2 and topvalue[1][0] == "":
    235 
    +        return _expand_expstr(content, content[topvalue[1][1]])
    236 
    +
    237 
    +    # Otherwise process fully...
    238 
    +    def internal_expand(value):
    239 
    +        (expansion_len, expansion_bits) = value
    240 
    +        idx = 0
    241 
    +        while idx < expansion_len:
    242 
    +            # First yield any constant string content
    243 
    +            yield expansion_bits[idx]
    244 
    +            idx += 1
    245 
    +            # Now, if there is an expansion variable left to expand, yield
    246 
    +            # the expansion of that variable too
    247 
    +            if idx < expansion_len:
    248 
    +                yield from internal_expand(content[expansion_bits[idx]])
    249 
    +            idx += 1
    250 
    +
    251 
    +    return "".join(internal_expand(topvalue))

  • buildstream/data/userconfig.yaml
    ... ... @@ -111,20 +111,3 @@ logging:
    111 111 
       message-format: |
    112 112
    113 113 
         [%{elapsed}][%{key}][%{element}] %{action} %{message}
    114 
    -
    115 
    -#
    116 
    -#    Prompt overrides
    117 
    -#
    118 
    -# Here you can suppress 'are you sure?' and other kinds of prompts by supplying
    119 
    -# override values. Note that e.g. 'yes' and 'no' have the same meaning here as
    120 
    -# they do in the actual cli prompt.
    121 
    -#
    122 
    -prompt:
    123 
    -
    124 
    -  # Whether to really proceed with 'bst workspace close' when doing so would
    125 
    -  # stop them from running bst commands in this workspace.
    126 
    -  #
    127 
    -  #  ask - Ask the user if they are sure.
    128 
    -  #  yes - Always close, without asking.
    129 
    -  #
    130 
    -  really-workspace-close-project-inaccessible: ask

  • buildstream/element.py
    ... ... @@ -894,10 +894,7 @@ class Element(Plugin):
    894 894 
                (str): The resolved value for *varname*, or None if no
    895 895 
                variable was declared with the given name.
    896 896 
             """
    897 
    -        if varname in self.__variables.variables:
    898 
    -            return self.__variables.variables[varname]
    899 
    -
    900 
    -        return None
    897 
    +        return self.__variables.flat.get(varname)
    901 898
    902 899 
         def batch_prepare_assemble(self, flags, *, collect=None):
    903 900 
             """ Configure command batching across prepare() and assemble()

  • buildstream/plugins/sources/tar.py
    ... ... @@ -154,7 +154,7 @@ class TarSource(DownloadableFileSource):
    154 154 
         # directory paths for the archived files.
    155 155 
         def _list_tar_paths(self, tar):
    156 156
    157 
    -        visited = {}
    157 
    +        visited = set()
    158 158 
             for member in tar.getmembers():
    159 159
    160 160 
                 # Remove any possible leading './', offer more consistent behavior
    ... ... @@ -170,7 +170,7 @@ class TarSource(DownloadableFileSource):
    170 170 
                     for i in range(len(components) - 1):
    171 171 
                         dir_component = '/'.join([components[j] for j in range(i + 1)])
    172 172 
                         if dir_component not in visited:
    173 
    -                        visited[dir_component] = True
    173 
    +                        visited.add(dir_component)
    174 174 
                             try:
    175 175 
                                 # Dont yield directory members which actually do
    176 176 
                                 # exist in the archive

  • buildstream/storage/_casbaseddirectory.py
    ... ... @@ -36,7 +36,7 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    36 36 
     from .._exceptions import BstError
    37 37 
     from .directory import Directory, VirtualDirectoryError
    38 38 
     from ._filebaseddirectory import FileBasedDirectory
    39 
    -from ..utils import FileListResult, safe_copy, list_relative_paths
    39 
    +from ..utils import FileListResult, safe_copy, list_relative_paths, _magic_timestamp
    40 40
    41 41
    42 42 
     class IndexEntry():
    ... ... @@ -535,7 +535,7 @@ class CasBasedDirectory(Directory):
    535 535 
                                     " The original error was: {}").
    536 536 
                                    format(src_name, entry.target, e))
    537 537
    538 
    -    def export_to_tar(self, tarfile, destination_dir, mtime=0):
    538 
    +    def export_to_tar(self, tarfile, destination_dir, mtime=_magic_timestamp):
    539 539 
             raise NotImplementedError()
    540 540
    541 541 
         def mark_changed(self):

  • buildstream/storage/_filebaseddirectory.py
    ... ... @@ -157,7 +157,7 @@ class FileBasedDirectory(Directory):
    157 157 
         # First, it sorts the results of os.listdir() to ensure the ordering of
    158 158 
         # the files in the archive is the same.  Second, it sets a fixed
    159 159 
         # timestamp for each entry. See also https://bugs.python.org/issue24465.
    160 
    -    def export_to_tar(self, tf, dir_arcname, mtime=0):
    160 
    +    def export_to_tar(self, tf, dir_arcname, mtime=_magic_timestamp):
    161 161 
             # We need directories here, including non-empty ones,
    162 162 
             # so list_relative_paths is not used.
    163 163 
             for filename in sorted(os.listdir(self.external_directory)):

  • buildstream/storage/directory.py
    ... ... @@ -32,6 +32,7 @@ See also: :ref:`sandboxing`.
    32 32 
     """
    33 33
    34 34 
     from .._exceptions import BstError, ErrorDomain
    35 
    +from ..utils import _magic_timestamp
    35 36
    36 37
    37 38 
     class VirtualDirectoryError(BstError):
    ... ... @@ -114,7 +115,7 @@ class Directory():
    114 115
    115 116 
             raise NotImplementedError()
    116 117
    117 
    -    def export_to_tar(self, tarfile, destination_dir, mtime=0):
    118 
    +    def export_to_tar(self, tarfile, destination_dir, mtime=_magic_timestamp):
    118 119 
             """ Exports this directory into the given tar file.
    119 120
    120 121 
             Args:

  • tests/frontend/buildcheckout.py
    ... ... @@ -252,6 +252,26 @@ def test_build_checkout_tarball_stdout(datafiles, cli):
    252 252 
         assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
    253 253
    254 254
    255 
    +@pytest.mark.datafiles(DATA_DIR)
    256 
    +def test_build_checkout_tarball_mtime_nonzero(datafiles, cli):
    257 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    258 
    +    tarpath = os.path.join(cli.directory, 'mtime_tar.tar')
    259 
    +
    260 
    +    result = cli.run(project=project, args=['build', 'target.bst'])
    261 
    +    result.assert_success()
    262 
    +
    263 
    +    checkout_args = ['artifact', 'checkout', '--tar', tarpath, 'target.bst']
    264 
    +    result = cli.run(project=project, args=checkout_args)
    265 
    +    result.assert_success()
    266 
    +
    267 
    +    tar = tarfile.TarFile(tarpath)
    268 
    +    for tarinfo in tar.getmembers():
    269 
    +        # An mtime of zero can be confusing to other software,
    270 
    +        # e.g. ninja build and template toolkit have both taken zero mtime to
    271 
    +        # mean 'file does not exist'.
    272 
    +        assert tarinfo.mtime > 0
    273 
    +
    274 
    +
    255 275 
     @pytest.mark.datafiles(DATA_DIR)
    256 276 
     def test_build_checkout_tarball_is_deterministic(datafiles, cli):
    257 277 
         project = os.path.join(datafiles.dirname, datafiles.basename)

  • tests/frontend/workspace.py
    ... ... @@ -1184,6 +1184,7 @@ def test_external_close_other(cli, datafiles, tmpdir_factory):
    1184 1184
    1185 1185 
         result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
    1186 1186 
         result.assert_success()
    1187 
    +    assert 'you can no longer run BuildStream' not in result.stderr
    1187 1188
    1188 1189
    1189 1190 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -1199,6 +1200,7 @@ def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element):
    1199 1200
    1200 1201 
         result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close'] + arg_elm)
    1201 1202 
         result.assert_success()
    1203 
    +    assert 'you can no longer run BuildStream' in result.stderr
    1202 1204
    1203 1205
    1204 1206 
     @pytest.mark.datafiles(DATA_DIR)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]