Angelos Evripiotis pushed to branch are_you_sure2 at BuildStream / buildstream
Commits:
-
0f78a47d
by Daniel Silverstone at 2019-02-15T10:03:46Z
-
14176e51
by Benjamin Schubert at 2019-02-15T11:04:10Z
-
39febfda
by Angelos Evripiotis at 2019-02-15T11:10:51Z
-
8b34e356
by Angelos Evripiotis at 2019-02-15T12:25:21Z
-
3bbb90e7
by Benjamin Schubert at 2019-02-15T12:34:36Z
-
0921ccf4
by Benjamin Schubert at 2019-02-15T13:39:52Z
-
c24f2971
by Angelos Evripiotis at 2019-02-15T13:56:48Z
14 changed files:
- buildstream/_context.py
- buildstream/_frontend/cli.py
- buildstream/_frontend/widget.py
- buildstream/_loader/loader.py
- buildstream/_profile.py
- buildstream/_variables.py
- buildstream/data/userconfig.yaml
- buildstream/element.py
- buildstream/plugins/sources/tar.py
- buildstream/storage/_casbaseddirectory.py
- buildstream/storage/_filebaseddirectory.py
- buildstream/storage/directory.py
- tests/frontend/buildcheckout.py
- tests/frontend/workspace.py
Changes:
... | ... | @@ -124,10 +124,6 @@ class Context(): |
124 | 124 |
# Whether or not to cache build trees on artifact creation
|
125 | 125 |
self.cache_buildtrees = None
|
126 | 126 |
|
127 |
- # Boolean, whether we double-check with the user that they meant to
|
|
128 |
- # close the workspace when they're using it to access the project.
|
|
129 |
- self.prompt_workspace_close_project_inaccessible = None
|
|
130 |
- |
|
131 | 127 |
# Whether elements must be rebuilt when their dependencies have changed
|
132 | 128 |
self._strict_build_plan = None
|
133 | 129 |
|
... | ... | @@ -248,22 +244,6 @@ class Context(): |
248 | 244 |
self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers')
|
249 | 245 |
self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries')
|
250 | 246 |
|
251 |
- # Load prompt preferences
|
|
252 |
- #
|
|
253 |
- # We convert string options to booleans here, so we can be both user
|
|
254 |
- # and coder-friendly. The string options are worded to match the
|
|
255 |
- # responses the user would give at the cli, for least surprise. The
|
|
256 |
- # booleans are converted here because it's easiest to eyeball that the
|
|
257 |
- # strings are right.
|
|
258 |
- #
|
|
259 |
- prompt = _yaml.node_get(
|
|
260 |
- defaults, Mapping, 'prompt')
|
|
261 |
- _yaml.node_validate(prompt, [
|
|
262 |
- 'really-workspace-close-project-inaccessible',
|
|
263 |
- ])
|
|
264 |
- self.prompt_workspace_close_project_inaccessible = _node_get_option_str(
|
|
265 |
- prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask'
|
|
266 |
- |
|
267 | 247 |
# Load per-projects overrides
|
268 | 248 |
self._project_overrides = _yaml.node_get(defaults, Mapping, 'projects', default_value={})
|
269 | 249 |
|
... | ... | @@ -814,6 +814,8 @@ def workspace_open(app, no_checkout, force, track_, directory, elements): |
814 | 814 |
def workspace_close(app, remove_dir, all_, elements):
|
815 | 815 |
"""Close a workspace"""
|
816 | 816 |
|
817 |
+ removed_required_element = False
|
|
818 |
+ |
|
817 | 819 |
with app.initialized():
|
818 | 820 |
if not (all_ or elements):
|
819 | 821 |
# NOTE: I may need to revisit this when implementing multiple projects
|
... | ... | @@ -840,18 +842,20 @@ def workspace_close(app, remove_dir, all_, elements): |
840 | 842 |
for element_name in elements:
|
841 | 843 |
if not app.stream.workspace_exists(element_name):
|
842 | 844 |
nonexisting.append(element_name)
|
843 |
- if (app.stream.workspace_is_required(element_name) and app.interactive and
|
|
844 |
- app.context.prompt_workspace_close_project_inaccessible):
|
|
845 |
- click.echo("Removing '{}' will prevent you from running "
|
|
846 |
- "BuildStream commands from the current directory".format(element_name))
|
|
847 |
- if not click.confirm('Are you sure you want to close this workspace?'):
|
|
848 |
- click.echo('Aborting', err=True)
|
|
849 |
- sys.exit(-1)
|
|
850 | 845 |
if nonexisting:
|
851 | 846 |
raise AppError("Workspace does not exist", detail="\n".join(nonexisting))
|
852 | 847 |
|
853 | 848 |
for element_name in elements:
|
854 | 849 |
app.stream.workspace_close(element_name, remove_dir=remove_dir)
|
850 |
+ if app.stream.workspace_is_required(element_name):
|
|
851 |
+ removed_required_element = True
|
|
852 |
+ |
|
853 |
+ # This message is echo'd last, as it's most relevant to the next
|
|
854 |
+ # thing the user will type.
|
|
855 |
+ if removed_required_element:
|
|
856 |
+ click.echo(
|
|
857 |
+ "Removed '{}', therefore you can no longer run BuildStream "
|
|
858 |
+ "commands from the current directory.".format(element_name), err=True)
|
|
855 | 859 |
|
856 | 860 |
|
857 | 861 |
##################################################################
|
... | ... | @@ -398,7 +398,7 @@ class LogLine(Widget): |
398 | 398 |
|
399 | 399 |
# Variables
|
400 | 400 |
if "%{vars" in format_:
|
401 |
- variables = _yaml.node_sanitize(element._Element__variables.variables)
|
|
401 |
+ variables = _yaml.node_sanitize(element._Element__variables.flat)
|
|
402 | 402 |
line = p.fmt_subst(
|
403 | 403 |
line, 'vars',
|
404 | 404 |
yaml.round_trip_dump(variables, default_flow_style=False, allow_unicode=True))
|
... | ... | @@ -284,17 +284,17 @@ class Loader(): |
284 | 284 |
def _check_circular_deps(self, element, check_elements=None, validated=None, sequence=None):
|
285 | 285 |
|
286 | 286 |
if check_elements is None:
|
287 |
- check_elements = {}
|
|
287 |
+ check_elements = set()
|
|
288 | 288 |
if validated is None:
|
289 |
- validated = {}
|
|
289 |
+ validated = set()
|
|
290 | 290 |
if sequence is None:
|
291 | 291 |
sequence = []
|
292 | 292 |
|
293 | 293 |
# Skip already validated branches
|
294 |
- if validated.get(element) is not None:
|
|
294 |
+ if element in validated:
|
|
295 | 295 |
return
|
296 | 296 |
|
297 |
- if check_elements.get(element) is not None:
|
|
297 |
+ if element in check_elements:
|
|
298 | 298 |
# Create `chain`, the loop of element dependencies from this
|
299 | 299 |
# element back to itself, by trimming everything before this
|
300 | 300 |
# element from the sequence under consideration.
|
... | ... | @@ -306,15 +306,15 @@ class Loader(): |
306 | 306 |
.format(element.full_name, " -> ".join(chain)))
|
307 | 307 |
|
308 | 308 |
# Push / Check each dependency / Pop
|
309 |
- check_elements[element] = True
|
|
309 |
+ check_elements.add(element)
|
|
310 | 310 |
sequence.append(element.full_name)
|
311 | 311 |
for dep in element.dependencies:
|
312 | 312 |
dep.element._loader._check_circular_deps(dep.element, check_elements, validated, sequence)
|
313 |
- del check_elements[element]
|
|
313 |
+ check_elements.remove(element)
|
|
314 | 314 |
sequence.pop()
|
315 | 315 |
|
316 | 316 |
# Eliminate duplicate paths
|
317 |
- validated[element] = True
|
|
317 |
+ validated.add(element)
|
|
318 | 318 |
|
319 | 319 |
# _sort_dependencies():
|
320 | 320 |
#
|
... | ... | @@ -26,7 +26,7 @@ import datetime |
26 | 26 |
import time
|
27 | 27 |
|
28 | 28 |
# Track what profile topics are active
|
29 |
-active_topics = {}
|
|
29 |
+active_topics = set()
|
|
30 | 30 |
active_profiles = {}
|
31 | 31 |
initialized = False
|
32 | 32 |
|
... | ... | @@ -144,14 +144,10 @@ def profile_init(): |
144 | 144 |
if setting:
|
145 | 145 |
topics = setting.split(':')
|
146 | 146 |
for topic in topics:
|
147 |
- active_topics[topic] = True
|
|
147 |
+ active_topics.add(topic)
|
|
148 | 148 |
initialized = True
|
149 | 149 |
|
150 | 150 |
|
151 | 151 |
def profile_enabled(topic):
|
152 | 152 |
profile_init()
|
153 |
- if active_topics.get(topic):
|
|
154 |
- return True
|
|
155 |
- if active_topics.get(Topics.ALL):
|
|
156 |
- return True
|
|
157 |
- return False
|
|
153 |
+ return topic in active_topics or Topics.ALL in active_topics
|
1 | 1 |
#
|
2 | 2 |
# Copyright (C) 2016 Codethink Limited
|
3 |
+# Copyright (C) 2019 Bloomberg L.P.
|
|
3 | 4 |
#
|
4 | 5 |
# This program is free software; you can redistribute it and/or
|
5 | 6 |
# modify it under the terms of the GNU Lesser General Public
|
... | ... | @@ -16,15 +17,37 @@ |
16 | 17 |
#
|
17 | 18 |
# Authors:
|
18 | 19 |
# Tristan Van Berkom <tristan vanberkom codethink co uk>
|
20 |
+# Daniel Silverstone <daniel silverstone codethink co uk>
|
|
19 | 21 |
|
20 | 22 |
import re
|
23 |
+import sys
|
|
21 | 24 |
|
22 | 25 |
from ._exceptions import LoadError, LoadErrorReason
|
23 | 26 |
from . import _yaml
|
24 | 27 |
|
25 | 28 |
# Variables are allowed to have dashes here
|
26 | 29 |
#
|
27 |
-_VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}'
|
|
30 |
+PARSE_EXPANSION = re.compile(r"\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}")
|
|
31 |
+ |
|
32 |
+ |
|
33 |
+# Throughout this code you will see variables named things like `expstr`.
|
|
34 |
+# These hold data structures called "expansion strings" and are the parsed
|
|
35 |
+# form of the strings which are the input to this subsystem. Strings
|
|
36 |
+# such as "Hello %{name}, how are you?" are parsed into the form:
|
|
37 |
+# (3, ["Hello ", "name", ", how are you?"])
|
|
38 |
+# i.e. a tuple of an integer and a list, where the integer is the cached
|
|
39 |
+# length of the list, and the list consists of one or more strings.
|
|
40 |
+# Strings in even indices of the list (0, 2, 4, etc) are constants which
|
|
41 |
+# are copied into the output of the expansion algorithm. Strings in the
|
|
42 |
+# odd indices (1, 3, 5, etc) are the names of further expansions to make.
|
|
43 |
+# In the example above, first "Hello " is copied, then "name" is expanded
|
|
44 |
+# and so must be another named expansion string passed in to the constructor
|
|
45 |
+# of the Variables class, and whatever is yielded from the expansion of "name"
|
|
46 |
+# is added to the concatenation for the result. Finally ", how are you?" is
|
|
47 |
+# copied in and the whole lot concatenated for return.
|
|
48 |
+#
|
|
49 |
+# To see how strings are parsed, see `_parse_expstr()` after the class, and
|
|
50 |
+# to see how expansion strings are expanded, see `_expand_expstr()` after that.
|
|
28 | 51 |
|
29 | 52 |
|
30 | 53 |
# The Variables helper object will resolve the variable references in
|
... | ... | @@ -38,14 +61,15 @@ _VARIABLE_MATCH = r'\%\{([a-zA-Z][a-zA-Z0-9_-]*)\}' |
38 | 61 |
# node (dict): A node loaded and composited with yaml tools
|
39 | 62 |
#
|
40 | 63 |
# Raises:
|
41 |
-# LoadError, if unresolved variables occur.
|
|
64 |
+# LoadError, if unresolved variables, or cycles in resolution, occur.
|
|
42 | 65 |
#
|
43 | 66 |
class Variables():
|
44 | 67 |
|
45 | 68 |
def __init__(self, node):
|
46 | 69 |
|
47 | 70 |
self.original = node
|
48 |
- self.variables = self._resolve(node)
|
|
71 |
+ self._expstr_map = self._resolve(node)
|
|
72 |
+ self.flat = self._flatten()
|
|
49 | 73 |
|
50 | 74 |
# subst():
|
51 | 75 |
#
|
... | ... | @@ -61,139 +85,167 @@ class Variables(): |
61 | 85 |
# LoadError, if the string contains unresolved variable references.
|
62 | 86 |
#
|
63 | 87 |
def subst(self, string):
|
64 |
- substitute, unmatched, _ = self._subst(string, self.variables)
|
|
65 |
- unmatched = list(set(unmatched))
|
|
66 |
- if unmatched:
|
|
67 |
- if len(unmatched) == 1:
|
|
68 |
- message = "Unresolved variable '{var}'".format(var=unmatched[0])
|
|
69 |
- else:
|
|
70 |
- message = "Unresolved variables: "
|
|
71 |
- for unmatch in unmatched:
|
|
72 |
- if unmatched.index(unmatch) > 0:
|
|
73 |
- message += ', '
|
|
74 |
- message += unmatch
|
|
75 |
- |
|
76 |
- raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
|
|
77 |
- |
|
78 |
- return substitute
|
|
79 |
- |
|
80 |
- def _subst(self, string, variables):
|
|
81 |
- |
|
82 |
- def subst_callback(match):
|
|
83 |
- nonlocal variables
|
|
84 |
- nonlocal unmatched
|
|
85 |
- nonlocal matched
|
|
86 |
- |
|
87 |
- token = match.group(0)
|
|
88 |
- varname = match.group(1)
|
|
89 |
- |
|
90 |
- value = _yaml.node_get(variables, str, varname, default_value=None)
|
|
91 |
- if value is not None:
|
|
92 |
- # We have to check if the inner string has variables
|
|
93 |
- # and return unmatches for those
|
|
94 |
- unmatched += re.findall(_VARIABLE_MATCH, value)
|
|
95 |
- matched += [varname]
|
|
96 |
- else:
|
|
97 |
- # Return unmodified token
|
|
98 |
- unmatched += [varname]
|
|
99 |
- value = token
|
|
100 |
- |
|
101 |
- return value
|
|
102 |
- |
|
103 |
- matched = []
|
|
104 |
- unmatched = []
|
|
105 |
- replacement = re.sub(_VARIABLE_MATCH, subst_callback, string)
|
|
106 |
- |
|
107 |
- return (replacement, unmatched, matched)
|
|
88 |
+ expstr = _parse_expstr(string)
|
|
89 |
+ |
|
90 |
+ try:
|
|
91 |
+ return _expand_expstr(self._expstr_map, expstr)
|
|
92 |
+ except KeyError:
|
|
93 |
+ unmatched = []
|
|
94 |
+ |
|
95 |
+ # Look for any unmatched variable names in the expansion string
|
|
96 |
+ for var in expstr[1][1::2]:
|
|
97 |
+ if var not in self._expstr_map:
|
|
98 |
+ unmatched.append(var)
|
|
99 |
+ |
|
100 |
+ if unmatched:
|
|
101 |
+ message = "Unresolved variable{}: {}".format(
|
|
102 |
+ "s" if len(unmatched) > 1 else "",
|
|
103 |
+ ", ".join(unmatched)
|
|
104 |
+ )
|
|
105 |
+ |
|
106 |
+ raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE, message)
|
|
107 |
+ # Otherwise, re-raise the KeyError since it clearly came from some
|
|
108 |
+ # other unknowable cause.
|
|
109 |
+ raise
|
|
108 | 110 |
|
109 | 111 |
# Variable resolving code
|
110 | 112 |
#
|
111 |
- # Here we substitute variables for values (resolve variables) repeatedly
|
|
112 |
- # in a dictionary, each time creating a new dictionary until there is no
|
|
113 |
- # more unresolved variables to resolve, or, until resolving further no
|
|
114 |
- # longer resolves anything, in which case we throw an exception.
|
|
113 |
+ # Here we resolve all of our inputs into a dictionary, ready for use
|
|
114 |
+ # in subst()
|
|
115 | 115 |
def _resolve(self, node):
|
116 |
- variables = node
|
|
117 |
- |
|
118 | 116 |
# Special case, if notparallel is specified in the variables for this
|
119 | 117 |
# element, then override max-jobs to be 1.
|
120 | 118 |
# Initialize it as a string as all variables are processed as strings.
|
121 | 119 |
#
|
122 |
- if _yaml.node_get(variables, bool, 'notparallel', default_value=False):
|
|
123 |
- variables['max-jobs'] = str(1)
|
|
124 |
- |
|
125 |
- # Resolve the dictionary once, reporting the new dictionary with things
|
|
126 |
- # substituted in it, and reporting unmatched tokens.
|
|
127 |
- #
|
|
128 |
- def resolve_one(variables):
|
|
129 |
- unmatched = []
|
|
130 |
- resolved = {}
|
|
131 |
- |
|
132 |
- for key, value in _yaml.node_items(variables):
|
|
133 |
- |
|
134 |
- # Ensure stringness of the value before substitution
|
|
135 |
- value = _yaml.node_get(variables, str, key)
|
|
136 |
- |
|
137 |
- resolved_var, item_unmatched, matched = self._subst(value, variables)
|
|
138 |
- |
|
139 |
- if _wrap_variable(key) in resolved_var:
|
|
140 |
- referenced_through = find_recursive_variable(key, matched, variables)
|
|
120 |
+ if _yaml.node_get(node, bool, 'notparallel', default_value=False):
|
|
121 |
+ node['max-jobs'] = str(1)
|
|
122 |
+ |
|
123 |
+ ret = {}
|
|
124 |
+ for key, value in _yaml.node_items(node):
|
|
125 |
+ value = _yaml.node_get(node, str, key)
|
|
126 |
+ ret[sys.intern(key)] = _parse_expstr(value)
|
|
127 |
+ return ret
|
|
128 |
+ |
|
129 |
+ def _check_for_missing(self):
|
|
130 |
+ # First the check for anything unresolvable
|
|
131 |
+ summary = []
|
|
132 |
+ for key, expstr in self._expstr_map.items():
|
|
133 |
+ for var in expstr[1][1::2]:
|
|
134 |
+ if var not in self._expstr_map:
|
|
135 |
+ line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}"
|
|
136 |
+ provenance = _yaml.node_get_provenance(self.original, key)
|
|
137 |
+ summary.append(line.format(unmatched=var, variable=key, provenance=provenance))
|
|
138 |
+ if summary:
|
|
139 |
+ raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
|
|
140 |
+ "Failed to resolve one or more variable:\n{}\n".format("\n".join(summary)))
|
|
141 |
+ |
|
142 |
+ def _check_for_cycles(self):
|
|
143 |
+ # And now the cycle checks
|
|
144 |
+ def cycle_check(expstr, visited, cleared):
|
|
145 |
+ for var in expstr[1][1::2]:
|
|
146 |
+ if var in cleared:
|
|
147 |
+ continue
|
|
148 |
+ if var in visited:
|
|
141 | 149 |
raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
|
142 |
- "{}: ".format(_yaml.node_get_provenance(variables, key)) +
|
|
150 |
+ "{}: ".format(_yaml.node_get_provenance(self.original, var)) +
|
|
143 | 151 |
("Variable '{}' expands to contain a reference to itself. " +
|
144 |
- "Perhaps '{}' contains '{}").format(key, referenced_through, _wrap_variable(key)))
|
|
145 |
- |
|
146 |
- resolved[key] = resolved_var
|
|
147 |
- unmatched += item_unmatched
|
|
148 |
- |
|
149 |
- # Carry over provenance
|
|
150 |
- resolved[_yaml.PROVENANCE_KEY] = variables[_yaml.PROVENANCE_KEY]
|
|
151 |
- return (resolved, unmatched)
|
|
152 |
- |
|
153 |
- # Resolve it until it's resolved or broken
|
|
154 |
- #
|
|
155 |
- resolved = variables
|
|
156 |
- unmatched = ['dummy']
|
|
157 |
- last_unmatched = ['dummy']
|
|
158 |
- while unmatched:
|
|
159 |
- resolved, unmatched = resolve_one(resolved)
|
|
160 |
- |
|
161 |
- # Lists of strings can be compared like this
|
|
162 |
- if unmatched == last_unmatched:
|
|
163 |
- # We've got the same result twice without matching everything,
|
|
164 |
- # something is undeclared or cyclic, compose a summary.
|
|
165 |
- #
|
|
166 |
- summary = ''
|
|
167 |
- for unmatch in set(unmatched):
|
|
168 |
- for var, provenance in self._find_references(unmatch):
|
|
169 |
- line = " unresolved variable '{unmatched}' in declaration of '{variable}' at: {provenance}\n"
|
|
170 |
- summary += line.format(unmatched=unmatch, variable=var, provenance=provenance)
|
|
171 |
- |
|
172 |
- raise LoadError(LoadErrorReason.UNRESOLVED_VARIABLE,
|
|
173 |
- "Failed to resolve one or more variable:\n{}".format(summary))
|
|
174 |
- |
|
175 |
- last_unmatched = unmatched
|
|
176 |
- |
|
177 |
- return resolved
|
|
178 |
- |
|
179 |
- # Helper function to fetch information about the node referring to a variable
|
|
152 |
+ "Perhaps '{}' contains '%{{{}}}").format(var, visited[-1], var))
|
|
153 |
+ visited.append(var)
|
|
154 |
+ cycle_check(self._expstr_map[var], visited, cleared)
|
|
155 |
+ visited.pop()
|
|
156 |
+ cleared.add(var)
|
|
157 |
+ |
|
158 |
+ cleared = set()
|
|
159 |
+ for key, expstr in self._expstr_map.items():
|
|
160 |
+ if key not in cleared:
|
|
161 |
+ cycle_check(expstr, [key], cleared)
|
|
162 |
+ |
|
163 |
+ # _flatten():
|
|
180 | 164 |
#
|
181 |
- def _find_references(self, varname):
|
|
182 |
- fullname = _wrap_variable(varname)
|
|
183 |
- for key, value in _yaml.node_items(self.original):
|
|
184 |
- if fullname in value:
|
|
185 |
- provenance = _yaml.node_get_provenance(self.original, key)
|
|
186 |
- yield (key, provenance)
|
|
187 |
- |
|
188 |
- |
|
189 |
-def find_recursive_variable(variable, matched_variables, all_vars):
|
|
190 |
- matched_values = (_yaml.node_get(all_vars, str, key) for key in matched_variables)
|
|
191 |
- for key, value in zip(matched_variables, matched_values):
|
|
192 |
- if _wrap_variable(variable) in value:
|
|
193 |
- return key
|
|
194 |
- # We failed to find a recursive variable
|
|
195 |
- return None
|
|
196 |
- |
|
197 |
- |
|
198 |
-def _wrap_variable(var):
|
|
199 |
- return "%{" + var + "}"
|
|
165 |
+ # Turn our dictionary of expansion strings into a flattened dict
|
|
166 |
+ # so that we can run expansions faster in the future
|
|
167 |
+ #
|
|
168 |
+ # Raises:
|
|
169 |
+ # LoadError, if the string contains unresolved variable references or
|
|
170 |
+ # if cycles are detected in the variable references
|
|
171 |
+ #
|
|
172 |
+ def _flatten(self):
|
|
173 |
+ flat = {}
|
|
174 |
+ try:
|
|
175 |
+ for key, expstr in self._expstr_map.items():
|
|
176 |
+ if expstr[0] > 1:
|
|
177 |
+ expstr = (1, [sys.intern(_expand_expstr(self._expstr_map, expstr))])
|
|
178 |
+ self._expstr_map[key] = expstr
|
|
179 |
+ flat[key] = expstr[1][0]
|
|
180 |
+ except KeyError:
|
|
181 |
+ self._check_for_missing()
|
|
182 |
+ raise
|
|
183 |
+ except RecursionError:
|
|
184 |
+ self._check_for_cycles()
|
|
185 |
+ raise
|
|
186 |
+ return flat
|
|
187 |
+ |
|
188 |
+ |
|
189 |
+# Cache for the parsed expansion strings. While this is nominally
|
|
190 |
+# something which might "waste" memory, in reality each of these
|
|
191 |
+# will live as long as the element which uses it, which is the
|
|
192 |
+# vast majority of the memory usage across the execution of BuildStream.
|
|
193 |
+PARSE_CACHE = {
|
|
194 |
+ # Prime the cache with the empty string since otherwise that can
|
|
195 |
+ # cause issues with the parser, complications to which cause slowdown
|
|
196 |
+ "": (1, [""]),
|
|
197 |
+}
|
|
198 |
+ |
|
199 |
+ |
|
200 |
+# Helper to parse a string into an expansion string tuple, caching
|
|
201 |
+# the results so that future parse requests don't need to think about
|
|
202 |
+# the string
|
|
203 |
+def _parse_expstr(instr):
|
|
204 |
+ try:
|
|
205 |
+ return PARSE_CACHE[instr]
|
|
206 |
+ except KeyError:
|
|
207 |
+ # This use of the regex turns a string like "foo %{bar} baz" into
|
|
208 |
+ # a list ["foo ", "bar", " baz"]
|
|
209 |
+ splits = PARSE_EXPANSION.split(instr)
|
|
210 |
+ # If an expansion ends the string, we get an empty string on the end
|
|
211 |
+ # which we can optimise away, making the expansion routines not need
|
|
212 |
+ # a test for this.
|
|
213 |
+ if splits[-1] == '':
|
|
214 |
+ splits = splits[:-1]
|
|
215 |
+ # Cache an interned copy of this. We intern it to try and reduce the
|
|
216 |
+ # memory impact of the cache. It seems odd to cache the list length
|
|
217 |
+ # but this is measurably cheaper than calculating it each time during
|
|
218 |
+ # string expansion.
|
|
219 |
+ PARSE_CACHE[instr] = (len(splits), [sys.intern(s) for s in splits])
|
|
220 |
+ return PARSE_CACHE[instr]
|
|
221 |
+ |
|
222 |
+ |
|
223 |
+# Helper to expand a given top level expansion string tuple in the context
|
|
224 |
+# of the given dictionary of expansion strings.
|
|
225 |
+#
|
|
226 |
+# Note: Will raise KeyError if any expansion is missing
|
|
227 |
+def _expand_expstr(content, topvalue):
|
|
228 |
+ # Short-circuit constant strings
|
|
229 |
+ if topvalue[0] == 1:
|
|
230 |
+ return topvalue[1][0]
|
|
231 |
+ |
|
232 |
+ # Short-circuit strings which are entirely an expansion of another variable
|
|
233 |
+ # e.g. "%{another}"
|
|
234 |
+ if topvalue[0] == 2 and topvalue[1][0] == "":
|
|
235 |
+ return _expand_expstr(content, content[topvalue[1][1]])
|
|
236 |
+ |
|
237 |
+ # Otherwise process fully...
|
|
238 |
+ def internal_expand(value):
|
|
239 |
+ (expansion_len, expansion_bits) = value
|
|
240 |
+ idx = 0
|
|
241 |
+ while idx < expansion_len:
|
|
242 |
+ # First yield any constant string content
|
|
243 |
+ yield expansion_bits[idx]
|
|
244 |
+ idx += 1
|
|
245 |
+ # Now, if there is an expansion variable left to expand, yield
|
|
246 |
+ # the expansion of that variable too
|
|
247 |
+ if idx < expansion_len:
|
|
248 |
+ yield from internal_expand(content[expansion_bits[idx]])
|
|
249 |
+ idx += 1
|
|
250 |
+ |
|
251 |
+ return "".join(internal_expand(topvalue))
|
... | ... | @@ -111,20 +111,3 @@ logging: |
111 | 111 |
message-format: |
|
112 | 112 |
|
113 | 113 |
[%{elapsed}][%{key}][%{element}] %{action} %{message}
|
114 |
- |
|
115 |
-#
|
|
116 |
-# Prompt overrides
|
|
117 |
-#
|
|
118 |
-# Here you can suppress 'are you sure?' and other kinds of prompts by supplying
|
|
119 |
-# override values. Note that e.g. 'yes' and 'no' have the same meaning here as
|
|
120 |
-# they do in the actual cli prompt.
|
|
121 |
-#
|
|
122 |
-prompt:
|
|
123 |
- |
|
124 |
- # Whether to really proceed with 'bst workspace close' when doing so would
|
|
125 |
- # stop them from running bst commands in this workspace.
|
|
126 |
- #
|
|
127 |
- # ask - Ask the user if they are sure.
|
|
128 |
- # yes - Always close, without asking.
|
|
129 |
- #
|
|
130 |
- really-workspace-close-project-inaccessible: ask
|
... | ... | @@ -894,10 +894,7 @@ class Element(Plugin): |
894 | 894 |
(str): The resolved value for *varname*, or None if no
|
895 | 895 |
variable was declared with the given name.
|
896 | 896 |
"""
|
897 |
- if varname in self.__variables.variables:
|
|
898 |
- return self.__variables.variables[varname]
|
|
899 |
- |
|
900 |
- return None
|
|
897 |
+ return self.__variables.flat.get(varname)
|
|
901 | 898 |
|
902 | 899 |
def batch_prepare_assemble(self, flags, *, collect=None):
|
903 | 900 |
""" Configure command batching across prepare() and assemble()
|
... | ... | @@ -154,7 +154,7 @@ class TarSource(DownloadableFileSource): |
154 | 154 |
# directory paths for the archived files.
|
155 | 155 |
def _list_tar_paths(self, tar):
|
156 | 156 |
|
157 |
- visited = {}
|
|
157 |
+ visited = set()
|
|
158 | 158 |
for member in tar.getmembers():
|
159 | 159 |
|
160 | 160 |
# Remove any possible leading './', offer more consistent behavior
|
... | ... | @@ -170,7 +170,7 @@ class TarSource(DownloadableFileSource): |
170 | 170 |
for i in range(len(components) - 1):
|
171 | 171 |
dir_component = '/'.join([components[j] for j in range(i + 1)])
|
172 | 172 |
if dir_component not in visited:
|
173 |
- visited[dir_component] = True
|
|
173 |
+ visited.add(dir_component)
|
|
174 | 174 |
try:
|
175 | 175 |
# Dont yield directory members which actually do
|
176 | 176 |
# exist in the archive
|
... | ... | @@ -36,7 +36,7 @@ from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2 |
36 | 36 |
from .._exceptions import BstError
|
37 | 37 |
from .directory import Directory, VirtualDirectoryError
|
38 | 38 |
from ._filebaseddirectory import FileBasedDirectory
|
39 |
-from ..utils import FileListResult, safe_copy, list_relative_paths
|
|
39 |
+from ..utils import FileListResult, safe_copy, list_relative_paths, _magic_timestamp
|
|
40 | 40 |
|
41 | 41 |
|
42 | 42 |
class IndexEntry():
|
... | ... | @@ -535,7 +535,7 @@ class CasBasedDirectory(Directory): |
535 | 535 |
" The original error was: {}").
|
536 | 536 |
format(src_name, entry.target, e))
|
537 | 537 |
|
538 |
- def export_to_tar(self, tarfile, destination_dir, mtime=0):
|
|
538 |
+ def export_to_tar(self, tarfile, destination_dir, mtime=_magic_timestamp):
|
|
539 | 539 |
raise NotImplementedError()
|
540 | 540 |
|
541 | 541 |
def mark_changed(self):
|
... | ... | @@ -157,7 +157,7 @@ class FileBasedDirectory(Directory): |
157 | 157 |
# First, it sorts the results of os.listdir() to ensure the ordering of
|
158 | 158 |
# the files in the archive is the same. Second, it sets a fixed
|
159 | 159 |
# timestamp for each entry. See also https://bugs.python.org/issue24465.
|
160 |
- def export_to_tar(self, tf, dir_arcname, mtime=0):
|
|
160 |
+ def export_to_tar(self, tf, dir_arcname, mtime=_magic_timestamp):
|
|
161 | 161 |
# We need directories here, including non-empty ones,
|
162 | 162 |
# so list_relative_paths is not used.
|
163 | 163 |
for filename in sorted(os.listdir(self.external_directory)):
|
... | ... | @@ -32,6 +32,7 @@ See also: :ref:`sandboxing`. |
32 | 32 |
"""
|
33 | 33 |
|
34 | 34 |
from .._exceptions import BstError, ErrorDomain
|
35 |
+from ..utils import _magic_timestamp
|
|
35 | 36 |
|
36 | 37 |
|
37 | 38 |
class VirtualDirectoryError(BstError):
|
... | ... | @@ -114,7 +115,7 @@ class Directory(): |
114 | 115 |
|
115 | 116 |
raise NotImplementedError()
|
116 | 117 |
|
117 |
- def export_to_tar(self, tarfile, destination_dir, mtime=0):
|
|
118 |
+ def export_to_tar(self, tarfile, destination_dir, mtime=_magic_timestamp):
|
|
118 | 119 |
""" Exports this directory into the given tar file.
|
119 | 120 |
|
120 | 121 |
Args:
|
... | ... | @@ -252,6 +252,26 @@ def test_build_checkout_tarball_stdout(datafiles, cli): |
252 | 252 |
assert os.path.join('.', 'usr', 'include', 'pony.h') in tar.getnames()
|
253 | 253 |
|
254 | 254 |
|
255 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
256 |
+def test_build_checkout_tarball_mtime_nonzero(datafiles, cli):
|
|
257 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
258 |
+ tarpath = os.path.join(cli.directory, 'mtime_tar.tar')
|
|
259 |
+ |
|
260 |
+ result = cli.run(project=project, args=['build', 'target.bst'])
|
|
261 |
+ result.assert_success()
|
|
262 |
+ |
|
263 |
+ checkout_args = ['artifact', 'checkout', '--tar', tarpath, 'target.bst']
|
|
264 |
+ result = cli.run(project=project, args=checkout_args)
|
|
265 |
+ result.assert_success()
|
|
266 |
+ |
|
267 |
+ tar = tarfile.TarFile(tarpath)
|
|
268 |
+ for tarinfo in tar.getmembers():
|
|
269 |
+ # An mtime of zero can be confusing to other software,
|
|
270 |
+ # e.g. ninja build and template toolkit have both taken zero mtime to
|
|
271 |
+ # mean 'file does not exist'.
|
|
272 |
+ assert tarinfo.mtime > 0
|
|
273 |
+ |
|
274 |
+ |
|
255 | 275 |
@pytest.mark.datafiles(DATA_DIR)
|
256 | 276 |
def test_build_checkout_tarball_is_deterministic(datafiles, cli):
|
257 | 277 |
project = os.path.join(datafiles.dirname, datafiles.basename)
|
... | ... | @@ -1184,6 +1184,7 @@ def test_external_close_other(cli, datafiles, tmpdir_factory): |
1184 | 1184 |
|
1185 | 1185 |
result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element])
|
1186 | 1186 |
result.assert_success()
|
1187 |
+ assert 'you can no longer run BuildStream' not in result.stderr
|
|
1187 | 1188 |
|
1188 | 1189 |
|
1189 | 1190 |
@pytest.mark.datafiles(DATA_DIR)
|
... | ... | @@ -1199,6 +1200,7 @@ def test_external_close_self(cli, datafiles, tmpdir_factory, guess_element): |
1199 | 1200 |
|
1200 | 1201 |
result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close'] + arg_elm)
|
1201 | 1202 |
result.assert_success()
|
1203 |
+ assert 'you can no longer run BuildStream' in result.stderr
|
|
1202 | 1204 |
|
1203 | 1205 |
|
1204 | 1206 |
@pytest.mark.datafiles(DATA_DIR)
|