Phillip Smyth pushed to branch caching_build_trees at BuildStream / buildstream
Commits:
8 changed files:
- buildstream/_artifactcache/artifactcache.py
- buildstream/_artifactcache/cascache.py
- buildstream/_frontend/cli.py
- buildstream/_stream.py
- buildstream/element.py
- + tests/integration/project/files/workspace-no-existing-cached-buildtree/Makefile
- + tests/integration/project/files/workspace-use-cached-buildtree/Makefile
- tests/integration/workspace.py
Changes:
... | ... | @@ -366,7 +366,7 @@ class ArtifactCache(): |
366 | 366 |
#
|
367 | 367 |
# Returns: path to extracted artifact
|
368 | 368 |
#
|
369 |
- def extract(self, element, key):
|
|
369 |
+ def extract(self, element, key, dest=None):
|
|
370 | 370 |
raise ImplError("Cache '{kind}' does not implement extract()"
|
371 | 371 |
.format(kind=type(self).__name__))
|
372 | 372 |
|
... | ... | @@ -483,6 +483,21 @@ class ArtifactCache(): |
483 | 483 |
raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
|
484 | 484 |
.format(kind=type(self).__name__))
|
485 | 485 |
|
486 |
+ # does get_buildtree_dir():
|
|
487 |
+ #
|
|
488 |
+ # Returns build tree from cache if exists
|
|
489 |
+ #
|
|
490 |
+ # Args:
|
|
491 |
+ # element (Element): The Element whose cache is being checked
|
|
492 |
+ # key: the related cache key
|
|
493 |
+ # directory: the directory to return if exists
|
|
494 |
+ # Returns:
|
|
495 |
+ # string: directory path or None
|
|
496 |
+ #
|
|
497 |
+ def get_buildtree_dir(self, element, key):
|
|
498 |
+ raise ImplError("Cache '{kind}' does not implement get_buildtree_dir()"
|
|
499 |
+ .format(kind=type(self).__name__))
|
|
500 |
+ |
|
486 | 501 |
################################################
|
487 | 502 |
# Local Private Methods #
|
488 | 503 |
################################################
|
... | ... | @@ -75,12 +75,14 @@ class CASCache(ArtifactCache): |
75 | 75 |
# This assumes that the repository doesn't have any dangling pointers
|
76 | 76 |
return os.path.exists(refpath)
|
77 | 77 |
|
78 |
- def extract(self, element, key):
|
|
78 |
+ def extract(self, element, key, dest=None):
|
|
79 | 79 |
ref = self.get_artifact_fullname(element, key)
|
80 | 80 |
|
81 | 81 |
tree = self.resolve_ref(ref, update_mtime=True)
|
82 | 82 |
|
83 |
- dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
|
|
83 |
+ if dest is None:
|
|
84 |
+ dest = os.path.join(self.extractdir, element._get_project().name, element.normal_name, tree.hash)
|
|
85 |
+ |
|
84 | 86 |
if os.path.isdir(dest):
|
85 | 87 |
# artifact has already been extracted
|
86 | 88 |
return dest
|
... | ... | @@ -106,6 +108,9 @@ class CASCache(ArtifactCache): |
106 | 108 |
|
107 | 109 |
return dest
|
108 | 110 |
|
111 |
+ def get_buildtree_dir(self, element, key):
|
|
112 |
+ return os.path.join(self.extract(element, key), "buildtree")
|
|
113 |
+ |
|
109 | 114 |
def commit(self, element, content, keys):
|
110 | 115 |
refs = [self.get_artifact_fullname(element, key) for key in keys]
|
111 | 116 |
|
... | ... | @@ -446,7 +446,9 @@ def pull(app, elements, deps, remote): |
446 | 446 |
all: All dependencies
|
447 | 447 |
"""
|
448 | 448 |
with app.initialized(session_name="Pull"):
|
449 |
- app.stream.pull(elements, selection=deps, remote=remote)
|
|
449 |
+ app.stream.pull(elements,
|
|
450 |
+ selection=deps,
|
|
451 |
+ remote=remote)
|
|
450 | 452 |
|
451 | 453 |
|
452 | 454 |
##################################################################
|
... | ... | @@ -681,11 +683,14 @@ def workspace(): |
681 | 683 |
help="Overwrite files existing in checkout directory")
|
682 | 684 |
@click.option('--track', 'track_', default=False, is_flag=True,
|
683 | 685 |
help="Track and fetch new source references before checking out the workspace")
|
686 |
+@click.option('--use-cached-buildtree', 'use_cached_buildtree', default='when-local',
|
|
687 |
+ type=click.Choice(['use-cached', 'ignore-cached', 'when-local']),
|
|
688 |
+ help="Using cached build trees")
|
|
684 | 689 |
@click.argument('element',
|
685 | 690 |
type=click.Path(readable=False))
|
686 | 691 |
@click.argument('directory', type=click.Path(file_okay=False))
|
687 | 692 |
@click.pass_obj
|
688 |
-def workspace_open(app, no_checkout, force, track_, element, directory):
|
|
693 |
+def workspace_open(app, no_checkout, force, track_, element, directory, use_cached_buildtree):
|
|
689 | 694 |
"""Open a workspace for manual source modification"""
|
690 | 695 |
|
691 | 696 |
if os.path.exists(directory):
|
... | ... | @@ -702,7 +707,8 @@ def workspace_open(app, no_checkout, force, track_, element, directory): |
702 | 707 |
app.stream.workspace_open(element, directory,
|
703 | 708 |
no_checkout=no_checkout,
|
704 | 709 |
track_first=track_,
|
705 |
- force=force)
|
|
710 |
+ force=force,
|
|
711 |
+ use_cached_buildtree=use_cached_buildtree)
|
|
706 | 712 |
|
707 | 713 |
|
708 | 714 |
##################################################################
|
... | ... | @@ -762,10 +768,13 @@ def workspace_close(app, remove_dir, all_, elements): |
762 | 768 |
help="Track and fetch the latest source before resetting")
|
763 | 769 |
@click.option('--all', '-a', 'all_', default=False, is_flag=True,
|
764 | 770 |
help="Reset all open workspaces")
|
771 |
+@click.option('--use-cached-buildtree', 'use_cached_buildtree', default='when-local',
|
|
772 |
+ type=click.Choice(['use-cached', 'ignore-cached', 'when-local']),
|
|
773 |
+ help="Using cached build trees")
|
|
765 | 774 |
@click.argument('elements', nargs=-1,
|
766 | 775 |
type=click.Path(readable=False))
|
767 | 776 |
@click.pass_obj
|
768 |
-def workspace_reset(app, soft, track_, all_, elements):
|
|
777 |
+def workspace_reset(app, soft, track_, all_, elements, use_cached_buildtree):
|
|
769 | 778 |
"""Reset a workspace to its original state"""
|
770 | 779 |
|
771 | 780 |
# Check that the workspaces in question exist
|
... | ... | @@ -785,7 +794,10 @@ def workspace_reset(app, soft, track_, all_, elements): |
785 | 794 |
if all_:
|
786 | 795 |
elements = tuple(element_name for element_name, _ in app.context.get_workspaces().list())
|
787 | 796 |
|
788 |
- app.stream.workspace_reset(elements, soft=soft, track_first=track_)
|
|
797 |
+ app.stream.workspace_reset(elements,
|
|
798 |
+ soft=soft,
|
|
799 |
+ track_first=track_,
|
|
800 |
+ use_cached_buildtree=use_cached_buildtree)
|
|
789 | 801 |
|
790 | 802 |
|
791 | 803 |
##################################################################
|
... | ... | @@ -435,6 +435,29 @@ class Stream(): |
435 | 435 |
raise StreamError("Error while staging dependencies into a sandbox"
|
436 | 436 |
": '{}'".format(e), detail=e.detail, reason=e.reason) from e
|
437 | 437 |
|
438 |
+ # __stage_cached_buildtree
|
|
439 |
+ #
|
|
440 |
+ # Stage a cached build tree if it exists
|
|
441 |
+ #
|
|
442 |
+ # Args:
|
|
443 |
+ # use_cached_buildtree (bool): Whether or not to use cached buildtrees
|
|
444 |
+ # element: element in use
|
|
445 |
+ # cache_activity, stage_activity, message: (str)
|
|
446 |
+ #
|
|
447 |
+ # Returns:
|
|
448 |
+ # (bool): True if the build tree was staged
|
|
449 |
+ #
|
|
450 |
+ def __stage_cached_buildtree(self, element, target_dir):
|
|
451 |
+ buildtree_path = None
|
|
452 |
+ if element._cached():
|
|
453 |
+ with element.timed_activity("Extracting cached build tree"):
|
|
454 |
+ buildtree_path = element._buildtree_path(element._get_cache_key())
|
|
455 |
+ if buildtree_path is not None:
|
|
456 |
+ with element.timed_activity("Staging cached build tree to {}".format(target_dir)):
|
|
457 |
+ shutil.copytree(buildtree_path, target_dir)
|
|
458 |
+ return True
|
|
459 |
+ return False
|
|
460 |
+ |
|
438 | 461 |
# workspace_open
|
439 | 462 |
#
|
440 | 463 |
# Open a project workspace
|
... | ... | @@ -445,11 +468,25 @@ class Stream(): |
445 | 468 |
# no_checkout (bool): Whether to skip checking out the source
|
446 | 469 |
# track_first (bool): Whether to track and fetch first
|
447 | 470 |
# force (bool): Whether to ignore contents in an existing directory
|
471 |
+ # use_cached_buildtree(str): Whether or not to use cached buildtrees
|
|
448 | 472 |
#
|
449 | 473 |
def workspace_open(self, target, directory, *,
|
450 | 474 |
no_checkout,
|
451 | 475 |
track_first,
|
452 |
- force):
|
|
476 |
+ force,
|
|
477 |
+ use_cached_buildtree):
|
|
478 |
+ |
|
479 |
+ workspaces = self._context.get_workspaces()
|
|
480 |
+ assert use_cached_buildtree in ('use-cached', 'when-local', 'ignore-cached')
|
|
481 |
+ |
|
482 |
+ # Make cached_buildtree a boolean based on the flag assigned to it
|
|
483 |
+ # If flag was `when-local`, assigning value based on whether or not the project uses a remote cache
|
|
484 |
+ if use_cached_buildtree == 'use-cached':
|
|
485 |
+ use_cached_buildtree = True
|
|
486 |
+ elif use_cached_buildtree == 'when-local' and not self._artifacts.has_fetch_remotes():
|
|
487 |
+ use_cached_buildtree = True
|
|
488 |
+ else:
|
|
489 |
+ use_cached_buildtree = False
|
|
453 | 490 |
|
454 | 491 |
if track_first:
|
455 | 492 |
track_targets = (target,)
|
... | ... | @@ -462,22 +499,6 @@ class Stream(): |
462 | 499 |
target = elements[0]
|
463 | 500 |
workdir = os.path.abspath(directory)
|
464 | 501 |
|
465 |
- if not list(target.sources()):
|
|
466 |
- build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
|
|
467 |
- if not build_depends:
|
|
468 |
- raise StreamError("The given element has no sources")
|
|
469 |
- detail = "Try opening a workspace on one of its dependencies instead:\n"
|
|
470 |
- detail += " \n".join(build_depends)
|
|
471 |
- raise StreamError("The given element has no sources", detail=detail)
|
|
472 |
- |
|
473 |
- workspaces = self._context.get_workspaces()
|
|
474 |
- |
|
475 |
- # Check for workspace config
|
|
476 |
- workspace = workspaces.get_workspace(target._get_full_name())
|
|
477 |
- if workspace and not force:
|
|
478 |
- raise StreamError("Workspace '{}' is already defined at: {}"
|
|
479 |
- .format(target.name, workspace.path))
|
|
480 |
- |
|
481 | 502 |
# If we're going to checkout, we need at least a fetch,
|
482 | 503 |
# if we were asked to track first, we're going to fetch anyway.
|
483 | 504 |
#
|
... | ... | @@ -487,6 +508,26 @@ class Stream(): |
487 | 508 |
track_elements = elements
|
488 | 509 |
self._fetch(elements, track_elements=track_elements)
|
489 | 510 |
|
511 |
+ # Check for workspace config
|
|
512 |
+ workspace = workspaces.get_workspace(target._get_full_name())
|
|
513 |
+ if workspace and not force:
|
|
514 |
+ raise StreamError("Workspace '{}' is already defined at: {}"
|
|
515 |
+ .format(target.name, workspace.path))
|
|
516 |
+ |
|
517 |
+ if use_cached_buildtree:
|
|
518 |
+ if self.__stage_cached_buildtree(target, workdir):
|
|
519 |
+ workspaces.save_config()
|
|
520 |
+ self._message(MessageType.INFO, "Saved workspace configuration")
|
|
521 |
+ return
|
|
522 |
+ |
|
523 |
+ if not list(target.sources()):
|
|
524 |
+ build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
|
|
525 |
+ if not build_depends:
|
|
526 |
+ raise StreamError("The given element has no sources")
|
|
527 |
+ detail = "Try opening a workspace on one of its dependencies instead:\n"
|
|
528 |
+ detail += " \n".join(build_depends)
|
|
529 |
+ raise StreamError("The given element has no sources", detail=detail)
|
|
530 |
+ |
|
490 | 531 |
if not no_checkout and target._get_consistency() != Consistency.CACHED:
|
491 | 532 |
raise StreamError("Could not stage uncached source. " +
|
492 | 533 |
"Use `--track` to track and " +
|
... | ... | @@ -547,8 +588,9 @@ class Stream(): |
547 | 588 |
# targets (list of str): The target elements to reset the workspace for
|
548 | 589 |
# soft (bool): Only reset workspace state
|
549 | 590 |
# track_first (bool): Whether to also track the sources first
|
591 |
+ # use_cached_buildtree(str): Whether or not to use cached buildtrees
|
|
550 | 592 |
#
|
551 |
- def workspace_reset(self, targets, *, soft, track_first):
|
|
593 |
+ def workspace_reset(self, targets, *, soft, track_first, use_cached_buildtree):
|
|
552 | 594 |
|
553 | 595 |
if track_first:
|
554 | 596 |
track_targets = targets
|
... | ... | @@ -592,6 +634,15 @@ class Stream(): |
592 | 634 |
workspaces.delete_workspace(element._get_full_name())
|
593 | 635 |
workspaces.create_workspace(element._get_full_name(), workspace.path)
|
594 | 636 |
|
637 |
+ if use_cached_buildtree == 'when-local':
|
|
638 |
+ use_cached_buildtree = os.path.isdir(os.path.join(workspace.path, 'buildtree'))
|
|
639 |
+ |
|
640 |
+ if use_cached_buildtree and self.__stage_cached_buildtree(element, workspace.path):
|
|
641 |
+ self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
|
|
642 |
+ .format(element.name, workspace.path))
|
|
643 |
+ return
|
|
644 |
+ |
|
645 |
+ workspaces.create_workspace(element.name, workspace.path)
|
|
595 | 646 |
with element.timed_activity("Staging sources to {}".format(workspace.path)):
|
596 | 647 |
element._open_workspace()
|
597 | 648 |
|
... | ... | @@ -893,6 +893,13 @@ class Element(Plugin): |
893 | 893 |
# Private Methods used in BuildStream #
|
894 | 894 |
#############################################################
|
895 | 895 |
|
896 |
+ # _buildtree_path():
|
|
897 |
+ #
|
|
898 |
+ # Returns the path of the cached build tree if it exists
|
|
899 |
+ #
|
|
900 |
+ def _buildtree_path(self, key):
|
|
901 |
+ return self.__artifacts.get_buildtree_dir(self, key)
|
|
902 |
+ |
|
896 | 903 |
# _new_from_meta():
|
897 | 904 |
#
|
898 | 905 |
# Recursively instantiate a new Element instance, it's sources
|
1 |
+test:
|
|
2 |
+ touch test.o
|
1 |
+test:
|
|
2 |
+ touch test.o
|
... | ... | @@ -5,6 +5,7 @@ from buildstream import _yaml |
5 | 5 |
from tests.testutils import cli_integration as cli
|
6 | 6 |
from tests.testutils.site import IS_LINUX
|
7 | 7 |
from tests.testutils.integration import walk_dir
|
8 |
+from tests.frontend import configure_project
|
|
8 | 9 |
|
9 | 10 |
|
10 | 11 |
pytestmark = pytest.mark.integration
|
... | ... | @@ -255,3 +256,119 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles): |
255 | 256 |
res = cli.run(project=project, args=['build', element_name])
|
256 | 257 |
res.assert_success()
|
257 | 258 |
assert not os.path.exists(os.path.join(workspace, 'prepared-again'))
|
259 |
+ |
|
260 |
+ |
|
261 |
+@pytest.mark.integration
|
|
262 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
263 |
+def test_use_cached_buildtree(cli, tmpdir, datafiles):
|
|
264 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
265 |
+ workspace = os.path.join(cli.directory, 'workspace')
|
|
266 |
+ element_path = os.path.join(project, 'elements')
|
|
267 |
+ element_name = 'workspace/workspace-use-cached-buildtree.bst'
|
|
268 |
+ |
|
269 |
+ element = {
|
|
270 |
+ 'kind': 'manual',
|
|
271 |
+ 'depends': [{
|
|
272 |
+ 'filename': 'base.bst',
|
|
273 |
+ 'type': 'build'
|
|
274 |
+ }],
|
|
275 |
+ 'sources': [{
|
|
276 |
+ 'kind': 'local',
|
|
277 |
+ 'path': 'files/workspace-use-cached-buildtree'
|
|
278 |
+ }],
|
|
279 |
+ 'config': {
|
|
280 |
+ 'build-commands': [
|
|
281 |
+ 'make'
|
|
282 |
+ ]
|
|
283 |
+ }
|
|
284 |
+ }
|
|
285 |
+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
|
|
286 |
+ _yaml.dump(element, os.path.join(element_path, element_name))
|
|
287 |
+ |
|
288 |
+ res = cli.run(project=project, args=['build', element_name])
|
|
289 |
+ res.assert_success()
|
|
290 |
+ |
|
291 |
+ res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
|
|
292 |
+ res.assert_success()
|
|
293 |
+ assert os.path.isdir(workspace)
|
|
294 |
+ assert os.path.exists(os.path.join(workspace, "test.o"))
|
|
295 |
+ |
|
296 |
+ |
|
297 |
+@pytest.mark.integration
|
|
298 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
299 |
+def test_dont_use_cached_buildtree(cli, tmpdir, datafiles):
|
|
300 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
301 |
+ workspace = os.path.join(cli.directory, 'workspace')
|
|
302 |
+ element_path = os.path.join(project, 'elements')
|
|
303 |
+ element_name = 'workspace/workspace-use-cached-buildtree.bst'
|
|
304 |
+ |
|
305 |
+ element = {
|
|
306 |
+ 'kind': 'manual',
|
|
307 |
+ 'depends': [{
|
|
308 |
+ 'filename': 'base.bst',
|
|
309 |
+ 'type': 'build'
|
|
310 |
+ }],
|
|
311 |
+ 'sources': [{
|
|
312 |
+ 'kind': 'local',
|
|
313 |
+ 'path': 'files/workspace-use-cached-buildtree'
|
|
314 |
+ }],
|
|
315 |
+ 'config': {
|
|
316 |
+ 'build-commands': [
|
|
317 |
+ 'make'
|
|
318 |
+ ]
|
|
319 |
+ }
|
|
320 |
+ }
|
|
321 |
+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
|
|
322 |
+ _yaml.dump(element, os.path.join(element_path, element_name))
|
|
323 |
+ |
|
324 |
+ res = cli.run(project=project, args=['build', element_name])
|
|
325 |
+ res.assert_success()
|
|
326 |
+ |
|
327 |
+ res = cli.run(project=project, args=['workspace', 'open',
|
|
328 |
+ '--use-cached-buildtree=ignore-cached',
|
|
329 |
+ element_name, workspace])
|
|
330 |
+ res.assert_success()
|
|
331 |
+ assert os.path.isdir(workspace)
|
|
332 |
+ assert not os.path.exists(os.path.join(workspace, "test.o"))
|
|
333 |
+ |
|
334 |
+ |
|
335 |
+@pytest.mark.integration
|
|
336 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
337 |
+# This tests the output if a build is done with the caching of build trees disabled
|
|
338 |
+# and then is reenabled and a workspace is opened
|
|
339 |
+def test_no_existing_cached_buildtree(cli, tmpdir, datafiles):
|
|
340 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
341 |
+ workspace = os.path.join(cli.directory, 'workspace')
|
|
342 |
+ element_path = os.path.join(project, 'elements')
|
|
343 |
+ element_name = 'workspace/workspace-no-existing-cached-buildtree.bst'
|
|
344 |
+ |
|
345 |
+ element = {
|
|
346 |
+ 'kind': 'manual',
|
|
347 |
+ 'depends': [{
|
|
348 |
+ 'filename': 'base.bst',
|
|
349 |
+ 'type': 'build'
|
|
350 |
+ }],
|
|
351 |
+ 'sources': [{
|
|
352 |
+ 'kind': 'local',
|
|
353 |
+ 'path': 'files/workspace-no-existing-cached-buildtree'
|
|
354 |
+ }],
|
|
355 |
+ 'config': {
|
|
356 |
+ 'build-commands': [
|
|
357 |
+ 'make'
|
|
358 |
+ ]
|
|
359 |
+ }
|
|
360 |
+ }
|
|
361 |
+ os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
|
|
362 |
+ _yaml.dump(element, os.path.join(element_path, element_name))
|
|
363 |
+ |
|
364 |
+ res = cli.run(project=project,
|
|
365 |
+ project_config={'variables': {'cache-buildtree': False}},
|
|
366 |
+ args=['build', element_name])
|
|
367 |
+ res.assert_success()
|
|
368 |
+ |
|
369 |
+ res = cli.run(project=project,
|
|
370 |
+ project_config={'variables': {'cache-buildtrees': True}},
|
|
371 |
+ args=['workspace', 'open', element_name, workspace])
|
|
372 |
+ res.assert_success()
|
|
373 |
+ assert os.path.isdir(workspace)
|
|
374 |
+ assert not os.path.exists(os.path.join(workspace, "test.o"))
|