Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream
Commits:
-
f8a54adc
by Tom Pollard at 2018-10-15T16:51:01Z
6 changed files:
- buildstream/_artifactcache/artifactcache.py
- buildstream/_artifactcache/cascache.py
- buildstream/_context.py
- buildstream/_frontend/cli.py
- buildstream/_stream.py
- buildstream/element.py
Changes:
... | ... | @@ -600,6 +600,21 @@ class ArtifactCache(): |
600 | 600 |
raise ImplError("Cache '{kind}' does not implement calculate_cache_size()"
|
601 | 601 |
.format(kind=type(self).__name__))
|
602 | 602 |
|
603 |
+ # checkout_artifact_subdir()
|
|
604 |
+ #
|
|
605 |
+ # Checkout given artifact subdir into provided directory
|
|
606 |
+ #
|
|
607 |
+ # Args:
|
|
608 |
+ # element (Element): The Element
|
|
609 |
+ # key (str): The cache key to use
|
|
610 |
+ # subdir (str): The subdir to checkout
|
|
611 |
+ # tmpdir (str): The dir to place the buildtree content
|
|
612 |
+ #
|
|
613 |
+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
|
|
614 |
+ raise ImplError("Cache '{kind}' does not implement checkout_artifact_subdir()"
|
|
615 |
+ .format(kind=type(self).__name__))
|
|
616 |
+ |
|
617 |
+ |
|
603 | 618 |
################################################
|
604 | 619 |
# Local Private Methods #
|
605 | 620 |
################################################
|
... | ... | @@ -439,6 +439,16 @@ class CASCache(ArtifactCache): |
439 | 439 |
|
440 | 440 |
return pushed
|
441 | 441 |
|
442 |
+ def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
|
|
443 |
+ tree = self.resolve_ref(self.get_artifact_fullname(element, key))
|
|
444 |
+ |
|
445 |
+ # This assumes that the subdir digest is present in the element tree
|
|
446 |
+ subdirdigest = self._get_subdir(tree, subdir)
|
|
447 |
+ |
|
448 |
+ with tmpdir:
|
|
449 |
+ self._checkout(tmpdir, subdirdigest)
|
|
450 |
+ |
|
451 |
+ |
|
442 | 452 |
################################################
|
443 | 453 |
# API Private Methods #
|
444 | 454 |
################################################
|
... | ... | @@ -113,6 +113,9 @@ class Context(): |
113 | 113 |
# Whether or not to attempt to pull buildtrees globally
|
114 | 114 |
self.pullbuildtrees = False
|
115 | 115 |
|
116 |
+ # Whether to not include artifact buildtrees in workspaces if available
|
|
117 |
+ self.workspacebuildtrees = True
|
|
118 |
+ |
|
116 | 119 |
# Private variables
|
117 | 120 |
self._cache_key = None
|
118 | 121 |
self._message_handler = None
|
... | ... | @@ -163,7 +166,7 @@ class Context(): |
163 | 166 |
_yaml.node_validate(defaults, [
|
164 | 167 |
'sourcedir', 'builddir', 'artifactdir', 'logdir',
|
165 | 168 |
'scheduler', 'artifacts', 'logging', 'projects',
|
166 |
- 'cache', 'pullbuildtrees'
|
|
169 |
+ 'cache', 'pullbuildtrees', 'workspacebuildtrees'
|
|
167 | 170 |
])
|
168 | 171 |
|
169 | 172 |
for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
|
... | ... | @@ -191,6 +194,9 @@ class Context(): |
191 | 194 |
# Load pull buildtrees configuration
|
192 | 195 |
self.pullbuildtrees = _yaml.node_get(defaults, bool, 'pullbuildtrees', default_value='False')
|
193 | 196 |
|
197 |
+ # Load workspace buildtrees configuration
|
|
198 |
+ self.workspacebuildtrees = _yaml.node_get(defaults, bool, 'workspacebuildtrees', default_value='True')
|
|
199 |
+ |
|
194 | 200 |
# Load logging config
|
195 | 201 |
logging = _yaml.node_get(defaults, Mapping, 'logging')
|
196 | 202 |
_yaml.node_validate(logging, [
|
... | ... | @@ -686,12 +686,16 @@ def workspace(): |
686 | 686 |
help="Overwrite files existing in checkout directory")
|
687 | 687 |
@click.option('--track', 'track_', default=False, is_flag=True,
|
688 | 688 |
help="Track and fetch new source references before checking out the workspace")
|
689 |
+@click.option('--no-cache', default=False, is_flag=True,
|
|
690 |
+ help="Do not checkout the cached buildtree")
|
|
689 | 691 |
@click.argument('element',
|
690 | 692 |
type=click.Path(readable=False))
|
691 | 693 |
@click.argument('directory', type=click.Path(file_okay=False))
|
692 | 694 |
@click.pass_obj
|
693 |
-def workspace_open(app, no_checkout, force, track_, element, directory):
|
|
694 |
- """Open a workspace for manual source modification"""
|
|
695 |
+def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
|
|
696 |
+ """Open a workspace for manual source modification, the elements buildtree
|
|
697 |
+ will be provided if available in the local artifact cache.
|
|
698 |
+ """
|
|
695 | 699 |
|
696 | 700 |
if os.path.exists(directory):
|
697 | 701 |
|
... | ... | @@ -703,11 +707,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory): |
703 | 707 |
click.echo("Checkout directory is not empty: {}".format(directory), err=True)
|
704 | 708 |
sys.exit(-1)
|
705 | 709 |
|
710 |
+ if not no_cache:
|
|
711 |
+ click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
|
|
712 |
+ |
|
706 | 713 |
with app.initialized():
|
707 | 714 |
app.stream.workspace_open(element, directory,
|
708 | 715 |
no_checkout=no_checkout,
|
709 | 716 |
track_first=track_,
|
710 |
- force=force)
|
|
717 |
+ force=force,
|
|
718 |
+ no_cache=no_cache)
|
|
711 | 719 |
|
712 | 720 |
|
713 | 721 |
##################################################################
|
... | ... | @@ -456,11 +456,17 @@ class Stream(): |
456 | 456 |
# no_checkout (bool): Whether to skip checking out the source
|
457 | 457 |
# track_first (bool): Whether to track and fetch first
|
458 | 458 |
# force (bool): Whether to ignore contents in an existing directory
|
459 |
+ # no_cache (bool): Whether to not include the cached buildtree
|
|
459 | 460 |
#
|
460 | 461 |
def workspace_open(self, target, directory, *,
|
461 | 462 |
no_checkout,
|
462 | 463 |
track_first,
|
463 |
- force):
|
|
464 |
+ force,
|
|
465 |
+ no_cache):
|
|
466 |
+ |
|
467 |
+ # Override no_cache if the global user conf workspacebuildtrees is false
|
|
468 |
+ if not self._context.workspacebuildtrees:
|
|
469 |
+ no_cache = True
|
|
464 | 470 |
|
465 | 471 |
if track_first:
|
466 | 472 |
track_targets = (target,)
|
... | ... | @@ -473,7 +479,21 @@ class Stream(): |
473 | 479 |
target = elements[0]
|
474 | 480 |
directory = os.path.abspath(directory)
|
475 | 481 |
|
476 |
- if not list(target.sources()):
|
|
482 |
+ # Check if given target has a buildtree artifact cached locally
|
|
483 |
+ buildtree = None
|
|
484 |
+ if target._cached():
|
|
485 |
+ buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
|
|
486 |
+ |
|
487 |
+ # If we're running in the default state, make the user aware of buildtree usage
|
|
488 |
+ if not no_cache:
|
|
489 |
+ if buildtree:
|
|
490 |
+ self._message(MessageType.INFO, "{} buildtree artifact is available,"
|
|
491 |
+ " workspace will be opened with it".format(target.name))
|
|
492 |
+ else:
|
|
493 |
+ self._message(MessageType.WARN, "{} buildtree artifact not available,"
|
|
494 |
+ " workspace will be opened with source checkout".format(target.name))
|
|
495 |
+ |
|
496 |
+ if (not buildtree or no_cache) and not list(target.sources()):
|
|
477 | 497 |
build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
|
478 | 498 |
if not build_depends:
|
479 | 499 |
raise StreamError("The given element has no sources")
|
... | ... | @@ -492,18 +512,21 @@ class Stream(): |
492 | 512 |
# If we're going to checkout, we need at least a fetch,
|
493 | 513 |
# if we were asked to track first, we're going to fetch anyway.
|
494 | 514 |
#
|
495 |
- if not no_checkout or track_first:
|
|
496 |
- track_elements = []
|
|
497 |
- if track_first:
|
|
498 |
- track_elements = elements
|
|
499 |
- self._fetch(elements, track_elements=track_elements)
|
|
500 |
- |
|
501 |
- if not no_checkout and target._get_consistency() != Consistency.CACHED:
|
|
502 |
- raise StreamError("Could not stage uncached source. " +
|
|
503 |
- "Use `--track` to track and " +
|
|
504 |
- "fetch the latest version of the " +
|
|
505 |
- "source.")
|
|
506 |
- |
|
515 |
+ if not buildtree or no_cache:
|
|
516 |
+ if not no_checkout or track_first:
|
|
517 |
+ track_elements = []
|
|
518 |
+ if track_first:
|
|
519 |
+ track_elements = elements
|
|
520 |
+ self._fetch(elements, track_elements=track_elements)
|
|
521 |
+ |
|
522 |
+ if not buildtree or no_cache:
|
|
523 |
+ if not no_checkout and target._get_consistency() != Consistency.CACHED:
|
|
524 |
+ raise StreamError("Could not stage uncached source. " +
|
|
525 |
+ "Use `--track` to track and " +
|
|
526 |
+ "fetch the latest version of the " +
|
|
527 |
+ "source.")
|
|
528 |
+ |
|
529 |
+ # Presume workspace to be forced if previous StreamError not raised
|
|
507 | 530 |
if workspace:
|
508 | 531 |
workspaces.delete_workspace(target._get_full_name())
|
509 | 532 |
workspaces.save_config()
|
... | ... | @@ -515,9 +538,14 @@ class Stream(): |
515 | 538 |
|
516 | 539 |
workspaces.create_workspace(target._get_full_name(), directory)
|
517 | 540 |
|
518 |
- if not no_checkout:
|
|
541 |
+ if (not buildtree or no_cache) and not no_checkout:
|
|
519 | 542 |
with target.timed_activity("Staging sources to {}".format(directory)):
|
520 | 543 |
target._open_workspace()
|
544 |
+ # Handle opening workspace with buildtree instead of source staging
|
|
545 |
+ elif buildtree and not no_cache:
|
|
546 |
+ with target.timed_activity("Staging buildtree to {}".format(directory)):
|
|
547 |
+ target._open_workspace(buildtree=buildtree)
|
|
548 |
+ |
|
521 | 549 |
|
522 | 550 |
workspaces.save_config()
|
523 | 551 |
self._message(MessageType.INFO, "Saved workspace configuration")
|
... | ... | @@ -1910,7 +1910,10 @@ class Element(Plugin): |
1910 | 1910 |
# This requires that a workspace already be created in
|
1911 | 1911 |
# the workspaces metadata first.
|
1912 | 1912 |
#
|
1913 |
- def _open_workspace(self):
|
|
1913 |
+ # Args:
|
|
1914 |
+ # buildtree (bool): Whether to open workspace with artifact buildtree
|
|
1915 |
+ #
|
|
1916 |
+ def _open_workspace(self, buildtree=None):
|
|
1914 | 1917 |
context = self._get_context()
|
1915 | 1918 |
workspace = self._get_workspace()
|
1916 | 1919 |
assert workspace is not None
|
... | ... | @@ -1923,11 +1926,15 @@ class Element(Plugin): |
1923 | 1926 |
# files in the target directory actually works without any
|
1924 | 1927 |
# additional support from Source implementations.
|
1925 | 1928 |
#
|
1929 |
+ |
|
1926 | 1930 |
os.makedirs(context.builddir, exist_ok=True)
|
1927 | 1931 |
with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
|
1928 | 1932 |
.format(self.normal_name)) as temp:
|
1929 |
- for source in self.sources():
|
|
1930 |
- source._init_workspace(temp)
|
|
1933 |
+ if not buildtree:
|
|
1934 |
+ for source in self.sources():
|
|
1935 |
+ source._init_workspace(temp)
|
|
1936 |
+ else:
|
|
1937 |
+ self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', temp)
|
|
1931 | 1938 |
|
1932 | 1939 |
# Now hardlink the files into the workspace target.
|
1933 | 1940 |
utils.link_files(temp, workspace.get_absolute_path())
|