GNOME.org
 

[Notes] [Git][BuildStream/buildstream][danielsilverstone-ct/variables-rework] 4 commits: unpack! fixme! reword! OMG!

Title: GitLab

Daniel Silverstone pushed to branch danielsilverstone-ct/variables-rework at BuildStream / buildstream

Commits:

3 changed files:

Changes:

  • buildstream/_frontend/widget.py
    ... ... @@ -398,7 +398,7 @@ class LogLine(Widget):
    398 398
    399 399 
                 # Variables
    400 400 
                 if "%{vars" in format_:
    401 
    -                variables = _yaml.node_sanitize(element._Element__variables.variables)
    401 
    +                variables = _yaml.node_sanitize(element._Element__variables.resolved())
    402 402 
                     line = p.fmt_subst(
    403 403 
                         line, 'vars',
    404 404 
                         yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))

  • buildstream/_variables.py
    1 1 
     #
    2 2 
     #  Copyright (C) 2016 Codethink Limited
    3 
    +#  Copyright (C) 2019 Bloomberg L.P.
    3 4 
     #
    4 5 
     #  This program is free software; you can redistribute it and/or
    5 6 
     #  modify it under the terms of the GNU Lesser General Public
    ... ... @@ -16,6 +17,7 @@
    16 17 
     #
    17 18 
     #  Authors:
    18 19 
     #        Tristan Van Berkom <tristan vanberkom codethink co uk>
    20 
    +#        Daniel Silverstone <daniel silverstone codethink co uk>
    19 21
    20 22 
     import re
    21 23 
     import sys
    ... ... @@ -25,7 +27,7 @@ from . import _yaml
    25 27
    26 28 
     # Variables are allowed to have dashes here
    27 29 
     #
    28 
    -_VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
    30 
    +PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
    29 31
    30 32
    31 33 
     # The Variables helper object will resolve the variable references in
    ... ... @@ -46,8 +48,14 @@ class Variables():
    46 48 
         def __init__(self, node):
    47 49
    48 50 
             self.original = node
    49 
    -#        self.variables = self._resolve(node)
    50 
    -        self.newexp = self._resolve_exp(node)
    51 
    +        self.newexp = self._resolve(node)
    52 
    +        self._sanity_check()
    53 
    +
    54 
    +    def get(self, var):
    55 
    +        if var in self.newexp:
    56 
    +            return _expand_expstr(self.newexp, self.newexp[var])
    57 
    +        else:
    58 
    +            return None
    51 59
    52 60 
         # subst():
    53 61 
         #
    ... ... @@ -63,203 +71,138 @@ class Variables():
    63 71 
         #    LoadError, if the string contains unresolved variable references.
    64 72 
         #
    65 73 
         def subst(self, string):
    66 
    -#        real = self.real_subst(string)
    67 
    -        exp = self.subst_exp(string)
    68 
    -#        if real != exp:
    69 
    -#            raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, "Oh dear!")
    70 
    -#        return real
    71 
    -        return exp
    72 
    -
    73 
    -    def real_subst(self, string):
    74 
    -        substitute, unmatched, _ = self._subst(string, self.variables)
    75 
    -        unmatched = list(set(unmatched))
    76 
    -        if unmatched:
    77 
    -            if len(unmatched) == 1:
    78 
    -                message = "Unresolved variable '{var}'".format(var=unmatched[0])
    79 
    -            else:
    80 
    -                message = "Unresolved variables: "
    81 
    -                for unmatch in unmatched:
    82 
    -                    if unmatched.index(unmatch) > 0:
    83 
    -                        message += ', '
    84 
    -                    message += unmatch
    85 
    -
    86 
    -            raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
    87 
    -
    88 
    -        return substitute
    89 
    -
    90 
    -    def _subst(self, string, variables):
    91 
    -
    92 
    -        def subst_callback(match):
    93 
    -            nonlocal variables
    94 
    -            nonlocal unmatched
    95 
    -            nonlocal matched
    96 
    -
    97 
    -            token = match.group(0)
    98 
    -            varname = match.group(1)
    74 
    +        exp = _parse_expstr(string)
    99 75
    100 
    -            value = _yaml.node_get(variables, str, varname, default_value=None)
    101 
    -            if value is not None:
    102 
    -                # We have to check if the inner string has variables
    103 
    -                # and return unmatches for those
    104 
    -                unmatched += re.findall(_VARIABLE_MATCH, value)
    105 
    -                matched += [varname]
    106 
    -            else:
    107 
    -                # Return unmodified token
    108 
    -                unmatched += [varname]
    109 
    -                value = token
    76 
    +        try:
    77 
    +            return _expand_expstr(self.newexp, exp)
    78 
    +        except KeyError:
    79 
    +            unmatched = []
    110 80
    111 
    -            return value
    81 
    +            for v in exp[1][1::2]:
    82 
    +                if v not in self.newexp:
    83 
    +                    unmatched.append(v)
    112 84
    113 
    -        matched = []
    114 
    -        unmatched = []
    115 
    -        replacement = re.sub(_VARIABLE_MATCH, subst_callback, string)
    85 
    +            if unmatched:
    86 
    +                if len(unmatched) == 1:
    87 
    +                    message = "Unresolved variable '{var}'".format(var=unmatched[0])
    88 
    +                else:
    89 
    +                    message = "Unresolved variables: "
    90 
    +                    for unmatch in unmatched:
    91 
    +                        if unmatched.index(unmatch) > 0:
    92 
    +                            message += ', '
    93 
    +                            message += unmatch
    116 94
    117 
    -        return (replacement, unmatched, matched)
    95 
    +                raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
    96 
    +            raise
    118 97
    119 98 
         # Variable resolving code
    120 99 
         #
    121 
    -    # Here we substitute variables for values (resolve variables) repeatedly
    122 
    -    # in a dictionary, each time creating a new dictionary until there is no
    123 
    -    # more unresolved variables to resolve, or, until resolving further no
    124 
    -    # longer resolves anything, in which case we throw an exception.
    100 
    +    # Here we resolve all of our inputs into a dictionary, ready for use
    101 
    +    # in subst()
    125 102 
         def _resolve(self, node):
    126 
    -        variables = node
    127 
    -
    128 103 
             # Special case, if notparallel is specified in the variables for this
    129 104 
             # element, then override max-jobs to be 1.
    130 105 
             # Initialize it as a string as all variables are processed as strings.
    131 106 
             #
    132 
    -        if _yaml.node_get(variables, bool, 'notparallel', default_value=False):
    133 
    -            variables['max-jobs'] = str(1)
    134 
    -
    135 
    -        # Resolve the dictionary once, reporting the new dictionary with things
    136 
    -        # substituted in it, and reporting unmatched tokens.
    137 
    -        #
    138 
    -        def resolve_one(variables):
    139 
    -            unmatched = []
    140 
    -            resolved = {}
    141 
    -
    142 
    -            for key, value in _yaml.node_items(variables):
    143 
    -
    144 
    -                # Ensure stringness of the value before substitution
    145 
    -                value = _yaml.node_get(variables, str, key)
    146 
    -
    147 
    -                resolved_var, item_unmatched, matched = self._subst(value, variables)
    148 
    -
    149 
    -                if _wrap_variable(key) in resolved_var:
    150 
    -                    referenced_through = find_recursive_variable(key, matched, variables)
    151 
    -                    raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
    152 
    -                                    "{}: ".format(_yaml.node_get_provenance(variables, key)) +
    153 
    -                                    ("Variable '{}' expands to contain a reference to itself. " +
    154 
    -                                     "Perhaps '{}' contains '{}").format(key, referenced_through, _wrap_variable(key)))
    155 
    -
    156 
    -                resolved[key] = resolved_var
    157 
    -                unmatched += item_unmatched
    158 
    -
    159 
    -            # Carry over provenance
    160 
    -            resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
    161 
    -            return (resolved, unmatched)
    162 
    -
    163 
    -        # Resolve it until it's resolved or broken
    164 
    -        #
    165 
    -        resolved = variables
    166 
    -        unmatched = ['dummy']
    167 
    -        last_unmatched = ['dummy']
    168 
    -        while unmatched:
    169 
    -            resolved, unmatched = resolve_one(resolved)
    107 
    +        if _yaml.node_get(node, bool, 'notparallel', default_value=False):
    108 
    +            node['max-jobs'] = str(1)
    170 109
    171 
    -            # Lists of strings can be compared like this
    172 
    -            if unmatched == last_unmatched:
    173 
    -                # We've got the same result twice without matching everything,
    174 
    -                # something is undeclared or cyclic, compose a summary.
    175 
    -                #
    176 
    -                summary = ''
    177 
    -                for unmatch in set(unmatched):
    178 
    -                    for var, provenance in self._find_references(unmatch):
    179 
    -                        line = "  unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}\n"
    180 
    -                        summary += line.format(unmatched=unmatch, variable=var, provenance=provenance)
    181 
    -
    182 
    -                raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
    183 
    -                                "Failed to resolve one or more variable:\n{}".format(summary))
    184 
    -
    185 
    -            last_unmatched = unmatched
    186 
    -
    187 
    -        return resolved
    188 
    -
    189 
    -    def _resolve_exp(self, node):
    190 110 
             ret = {}
    191 111 
             for key, value in _yaml.node_items(node):
    192 112 
                 value = _yaml.node_get(node, str, key)
    193 113 
                 ret[sys.intern(key)] = _parse_expstr(value)
    194 114 
             return ret
    195 115
    196 
    -    def subst_exp(self, string):
    197 
    -        exp = _parse_expstr(string)
    198 
    -        return _expand_expstr(self.newexp, exp)
    199 
    -
    200 
    -    # Helper function to fetch information about the node referring to a variable
    116 
    +    # Sanity checks on the resolved variables dictionary
    201 117 
         #
    202 
    -    def _find_references(self, varname):
    203 
    -        fullname = _wrap_variable(varname)
    204 
    -        for key, value in _yaml.node_items(self.original):
    205 
    -            if fullname in value:
    206 
    -                provenance = _yaml.node_get_provenance(self.original, key)
    207 
    -                yield (key, provenance)
    208 
    -
    209 
    -
    210 
    -def find_recursive_variable(variable, matched_variables, all_vars):
    211 
    -    matched_values = (_yaml.node_get(all_vars, str, key) for key in matched_variables)
    212 
    -    for key, value in zip(matched_variables, matched_values):
    213 
    -        if _wrap_variable(variable) in value:
    214 
    -            return key
    215 
    -    # We failed to find a recursive variable
    216 
    -    return None
    217 
    -
    218 
    -
    219 
    -def _wrap_variable(var):
    220 
    -    return "%{" + var + "}"
    221 
    -
    222 
    -## Stuff for new style expansion strings
    223 
    -
    224 
    -PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
    118 
    +    # Here we check for loops and for missing inputs which might cause the
    119 
    +    # resolved dictionary of inputs to not be sufficient.
    120 
    +    def _sanity_check(self):
    121 
    +        # First the check for anything unresolvable
    122 
    +        summary = []
    123 
    +        wants = {}
    124 
    +        for k, es in self.newexp.items():
    125 
    +            wants[k] = wants.get(k, set())
    126 
    +            for var in es[1][1::2]:
    127 
    +                wants[k].add(var)
    128 
    +                if var not in self.newexp:
    129 
    +                    line = "  unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
    130 
    +                    provenance = _yaml.node_get_provenance(self.original, k)
    131 
    +                    summary.append(line.format(unmatched=var, variable=k, provenance=provenance))
    132 
    +        if summary:
    133 
    +            raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
    134 
    +                            "Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
    135 
    +
    136 
    +        # And now the cycle checks
    137 
    +        def cycle_check(exp, visited, cleared):
    138 
    +            for var in exp[1][1::2]:
    139 
    +                if var in cleared:
    140 
    +                    continue
    141 
    +                if var in visited:
    142 
    +                    raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
    143 
    +                                    "{}: ".format(_yaml.node_get_provenance(self.original, var)) +
    144 
    +                                    ("Variable '{}' expands to contain a reference to itself. " +
    145 
    +                                     "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
    146 
    +                visited.append(var)
    147 
    +                cycle_check(self.newexp[var], visited, cleared)
    148 
    +                visited.pop()
    149 
    +                cleared.add(var)
    150 
    +
    151 
    +        cleared = set()
    152 
    +        for k, es in self.newexp.items():
    153 
    +            if k not in cleared:
    154 
    +                cycle_check(es, [k], cleared)
    155 
    +
    156 
    +    # resolved():
    157 
    +    #
    158 
    +    # Perform any substitutions necessary and return a dictionary of all
    159 
    +    # inputs fully resolved.
    160 
    +    #
    161 
    +    # Returns:
    162 
    +    #    (dict): A dictionary mapping variable name to resolved content
    163 
    +    #
    164 
    +    def resolved(self):
    165 
    +        # Make a fresh dict of the resolved expansions
    166 
    +        ret = {}
    167 
    +        for k, es in self.newexp.items():
    168 
    +            ret[k] = _expand_expstr(self.newexp, es)
    169 
    +        return ret
    225 170
    226 
    -def __parse_expstr(s):
    227 
    -    spl = PARSE_EXPANSION.split(s)
    228 
    -    if spl[-1] == '':
    229 
    -        spl = spl[:-1]
    230 
    -    return (len(spl), [sys.intern(s) for s in spl])
    231 171
    172 
    +# Cache for the parsed expansion strings.  While this is nominally
    173 
    +# something which might "waste" memory, in reality each of these
    174 
    +# will live as long as the element which uses it, which is the
    175 
    +# vast majority of the memory usage across the execution of BuildStream.
    232 176 
     PARSE_CACHE = {}
    233 177
    234 
    -def _parse_expstr(s):
    178 
    +
    179 
    +# Helper to parse a string into an expansion string tuple, caching
    180 
    +# the results so that future parse requests don't need to think about
    181 
    +# the string
    182 
    +def _parse_expstr(instr):
    235 183 
         try:
    236 
    -        return PARSE_CACHE[s]
    184 
    +        return PARSE_CACHE[instr]
    237 185 
         except KeyError:
    238 
    -        PARSE_CACHE[s] = __parse_expstr(s)
    239 
    -        return PARSE_CACHE[s]
    186 
    +        spl = PARSE_EXPANSION.split(instr)
    187 
    +        if spl[-1] == '':
    188 
    +            spl = spl[:-1]
    189 
    +        PARSE_CACHE[instr] = (len(spl), [sys.intern(s) for s in spl])
    190 
    +        return PARSE_CACHE[instr]
    240 191
    241 
    -#def _expand_expstr(content, value):
    242 
    -#    ret = []
    243 
    -#    expstack = [(value,0)]
    244 
    -#    while expstack:
    245 
    -#        ((elen,expanded), idx) = expstack.pop()
    246 
    -#        ret.append(expanded[idx])
    247 
    -#        idx += 1
    248 
    -#        if idx < elen:
    249 
    -#            if elen > idx+1:
    250 
    -#                expstack.append(((elen,expanded), idx+1))
    251 
    -#            expstack.append((content[expanded[idx]], 0))
    252 
    -#    return "".join(ret)
    253 192
    254 
    -def __expand(content, value):
    255 
    -    (elen, bits) = value
    256 
    -    idx = 0
    257 
    -    while idx < elen:
    258 
    -        yield bits[idx]
    259 
    -        idx += 1
    260 
    -        if idx < elen:
    261 
    -            yield from __expand(content, content[bits[idx]])
    262 
    -        idx += 1
    263 
    -
    264 
    -def _expand_expstr(content, value):
    265 
    -    return "".join(__expand(content, value))
    193 
    +# Helper to expand a given top level expansion string tuple in the context
    194 
    +# of the given dictionary of expansion strings.
    195 
    +#
    196 
    +# Note: Will raise KeyError if any expansion is missing
    197 
    +def _expand_expstr(content, topvalue):
    198 
    +    def __expand(value):
    199 
    +        (elen, bits) = value
    200 
    +        idx = 0
    201 
    +        while idx < elen:
    202 
    +            yield bits[idx]
    203 
    +            idx += 1
    204 
    +            if idx < elen:
    205 
    +                yield from __expand(content[bits[idx]])
    206 
    +            idx += 1
    207 
    +
    208 
    +    return "".join(__expand(topvalue))

  • buildstream/element.py
    ... ... @@ -894,10 +894,7 @@ class Element(Plugin):
    894 894 
                (str): The resolved value for *varname*, or None if no
    895 895 
                variable was declared with the given name.
    896 896 
             """
    897 
    -        if varname in self.__variables.variables:
    898 
    -            return self.__variables.variables[varname]
    899 
    -
    900 
    -        return None
    897 
    +        return self.__variables.get(varname)
    901 898
    902 899 
         def batch_prepare_assemble(self, flags, *, collect=None):
    903 900 
             """ Configure command batching across prepare() and assemble()

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]