GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/workspacebuildtree] WIP: Opening a workspace with a cached build

Title: GitLab

Tom Pollard pushed to branch tpollard/workspacebuildtree at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/_artifactcache/artifactcache.py
    ... ... @@ -835,6 +835,20 @@ class ArtifactCache():
    835 835
    836 836 
             self.cas.link_ref(oldref, newref)
    837 837
    838 
    +    # checkout_artifact_subdir()
    839 
    +    #
    840 
    +    # Checkout given artifact subdir into provided directory
    841 
    +    #
    842 
    +    # Args:
    843 
    +    #     element (Element): The Element
    844 
    +    #     key (str): The cache key to use
    845 
    +    #     subdir (str): The subdir to checkout
    846 
    +    #     tmpdir (str): The dir to place the subdir content
    847 
    +    #
    848 
    +    def checkout_artifact_subdir(self, element, key, subdir, tmpdir):
    849 
    +        ref = self.get_artifact_fullname(element, key)
    850 
    +        return self.cas.checkout_artifact_subdir(ref, subdir, tmpdir)
    851 
    +
    838 852 
         ################################################
    839 853 
         #               Local Private Methods          #
    840 854 
         ################################################

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -404,6 +404,21 @@ class CASCache():
    404 404
    405 405 
             return True
    406 406
    407 
    +    # checkout_artifact_subdir():
    408 
    +    #
    409 
    +    # Checkout given artifact subdir into provided directory
    410 
    +    #
    411 
    +    # Args:
    412 
    +    #     ref (str): The ref to check
    413 
    +    #     subdir (str): The subdir to checkout
    414 
    +    #     tmpdir (str): The dir to place the subdir content
    415 
    +    #
    416 
    +    def checkout_artifact_subdir(self, ref, subdir, tmpdir):
    417 
    +        tree = self.resolve_ref(ref)
    418 
    +        # This assumes that the subdir digest is present in the element tree
    419 
    +        subdirdigest = self._get_subdir(tree, subdir)
    420 
    +        self._checkout(tmpdir, subdirdigest)
    421 
    +
    407 422 
         # objpath():
    408 423 
         #
    409 424 
         # Return the path of an object based on its digest.

  • buildstream/_context.py
    ... ... @@ -122,6 +122,9 @@ class Context():
    122 122 
             # a hard reset of a workspace, potentially losing changes.
    123 123 
             self.prompt_workspace_reset_hard = None
    124 124
    125 
    +        # Whether to not include artifact buildtrees in workspaces if available
    126 
    +        self.workspace_buildtrees = True
    127 
    +
    125 128 
             # Whether elements must be rebuilt when their dependencies have changed
    126 129 
             self._strict_build_plan = None
    127 130
    ... ... @@ -177,7 +180,7 @@ class Context():
    177 180 
             _yaml.node_validate(defaults, [
    178 181 
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    179 182 
                 'scheduler', 'artifacts', 'logging', 'projects',
    180 
    -            'cache', 'prompt'
    183 
    +            'cache', 'prompt', 'workspace-buildtrees'
    181 184 
             ])
    182 185
    183 186 
             for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    ... ... @@ -205,6 +208,9 @@ class Context():
    205 208 
             # Load pull build trees configuration
    206 209 
             self.pull_buildtrees = _yaml.node_get(cache, bool, 'pull-buildtrees')
    207 210
    211 
    +        # Load workspace buildtrees configuration
    212 
    +        self.workspace_buildtrees = _yaml.node_get(defaults, bool, 'workspace-buildtrees', default_value='True')
    213 
    +
    208 214 
             # Load logging config
    209 215 
             logging = _yaml.node_get(defaults, Mapping, 'logging')
    210 216 
             _yaml.node_validate(logging, [

  • buildstream/_frontend/cli.py
    ... ... @@ -705,17 +705,21 @@ def workspace():
    705 705 
     ##################################################################
    706 706 
     @workspace.command(name='open', short_help="Open a new workspace")
    707 707 
     @click.option('--no-checkout', default=False, is_flag=True,
    708 
    -              help="Do not checkout the source, only link to the given directory")
    708 
    +              help="Do not checkout the source or cached buildtree, only link to the given directory")
    709 709 
     @click.option('--force', '-f', default=False, is_flag=True,
    710 710 
                   help="Overwrite files existing in checkout directory")
    711 711 
     @click.option('--track', 'track_', default=False, is_flag=True,
    712 712 
                   help="Track and fetch new source references before checking out the workspace")
    713 
    +@click.option('--no-cache', default=False, is_flag=True,
    714 
    +              help="Do not checkout the cached buildtree")
    713 715 
     @click.argument('element',
    714 716 
                     type=click.Path(readable=False))
    715 717 
     @click.argument('directory', type=click.Path(file_okay=False))
    716 718 
     @click.pass_obj
    717 
    -def workspace_open(app, no_checkout, force, track_, element, directory):
    718 
    -    """Open a workspace for manual source modification"""
    719 
    +def workspace_open(app, no_checkout, force, track_, no_cache, element, directory):
    720 
    +    """Open a workspace for manual source modification, the elements buildtree
    721 
    +    will be provided if available in the local artifact cache.
    722 
    +    """
    719 723
    720 724 
         if os.path.exists(directory):
    721 725
    ... ... @@ -727,11 +731,15 @@ def workspace_open(app, no_checkout, force, track_, element, directory):
    727 731 
                 click.echo("Checkout directory is not empty: {}".format(directory), err=True)
    728 732 
                 sys.exit(-1)
    729 733
    734 
    +    if not no_cache and not no_checkout:
    735 
    +        click.echo("WARNING: Workspace will be opened without the cached buildtree if not cached locally")
    736 
    +
    730 737 
         with app.initialized():
    731 738 
             app.stream.workspace_open(element, directory,
    732 739 
                                       no_checkout=no_checkout,
    733 740 
                                       track_first=track_,
    734 
    -                                  force=force)
    741 
    +                                  force=force,
    742 
    +                                  no_cache=no_cache)
    735 743
    736 744
    737 745 
     ##################################################################

  • buildstream/_stream.py
    ... ... @@ -469,11 +469,17 @@ class Stream():
    469 469 
         #    no_checkout (bool): Whether to skip checking out the source
    470 470 
         #    track_first (bool): Whether to track and fetch first
    471 471 
         #    force (bool): Whether to ignore contents in an existing directory
    472 
    +    #    no_cache (bool): Whether to not include the cached buildtree
    472 473 
         #
    473 474 
         def workspace_open(self, target, directory, *,
    474 475 
                            no_checkout,
    475 476 
                            track_first,
    476 
    -                       force):
    477 
    +                       force,
    478 
    +                       no_cache):
    479 
    +
    480 
    +        # Set no_cache if the global user conf workspacebuildtrees is false
    481 
    +        if not self._context.workspace_buildtrees:
    482 
    +            no_cache = True
    477 483
    478 484 
             if track_first:
    479 485 
                 track_targets = (target,)
    ... ... @@ -486,6 +492,20 @@ class Stream():
    486 492 
             target = elements[0]
    487 493 
             directory = os.path.abspath(directory)
    488 494
    495 
    +        # Check if given target has a buildtree artifact cached locally
    496 
    +        buildtree = None
    497 
    +        if target._cached():
    498 
    +            buildtree = self._artifacts.contains_subdir_artifact(target, target._get_cache_key(), 'buildtree')
    499 
    +
    500 
    +        # If we're running in the default state, make the user aware of buildtree usage
    501 
    +        if not no_cache and not no_checkout:
    502 
    +            if buildtree:
    503 
    +                self._message(MessageType.INFO, "{} buildtree artifact is available,"
    504 
    +                              " workspace will be opened with it".format(target.name))
    505 
    +            else:
    506 
    +                self._message(MessageType.WARN, "{} buildtree artifact not available,"
    507 
    +                              " workspace will be opened with source checkout".format(target.name))
    508 
    +
    489 509 
             if not list(target.sources()):
    490 510 
                 build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    491 511 
                 if not build_depends:
    ... ... @@ -517,6 +537,7 @@ class Stream():
    517 537 
                                   "fetch the latest version of the " +
    518 538 
                                   "source.")
    519 539
    540 
    +        # Presume workspace to be forced if previous StreamError not raised
    520 541 
             if workspace:
    521 542 
                 workspaces.delete_workspace(target._get_full_name())
    522 543 
                 workspaces.save_config()
    ... ... @@ -526,11 +547,16 @@ class Stream():
    526 547 
             except OSError as e:
    527 548 
                 raise StreamError("Failed to create workspace directory: {}".format(e)) from e
    528 549
    529 
    -        workspaces.create_workspace(target._get_full_name(), directory)
    530 
    -
    531 
    -        if not no_checkout:
    532 
    -            with target.timed_activity("Staging sources to {}".format(directory)):
    533 
    -                target._open_workspace()
    550 
    +        # Handle opening workspace with buildtree included
    551 
    +        if (buildtree and not no_cache) and not no_checkout:
    552 
    +            workspaces.create_workspace(target._get_full_name(), directory, cached_build=buildtree)
    553 
    +            with target.timed_activity("Staging buildtree to {}".format(directory)):
    554 
    +                target._open_workspace(buildtree=buildtree)
    555 
    +        else:
    556 
    +            workspaces.create_workspace(target._get_full_name(), directory)
    557 
    +            if (not buildtree or no_cache) and not no_checkout:
    558 
    +                with target.timed_activity("Staging sources to {}".format(directory)):
    559 
    +                    target._open_workspace()
    534 560
    535 561 
             workspaces.save_config()
    536 562 
             self._message(MessageType.INFO, "Saved workspace configuration")
    ... ... @@ -614,10 +640,24 @@ class Stream():
    614 640 
                                           .format(workspace_path, e)) from e
    615 641
    616 642 
                 workspaces.delete_workspace(element._get_full_name())
    617 
    -            workspaces.create_workspace(element._get_full_name(), workspace_path)
    618 643
    619 
    -            with element.timed_activity("Staging sources to {}".format(workspace_path)):
    620 
    -                element._open_workspace()
    644 
    +            # Create the workspace, ensuring the original optional cached build state is preserved if
    645 
    +            # possible.
    646 
    +            buildtree = False
    647 
    +            if workspace.cached_build and element._cached():
    648 
    +                if self._artifacts.contains_subdir_artifact(element, element._get_cache_key(), 'buildtree'):
    649 
    +                    buildtree = True
    650 
    +
    651 
    +            # Warn the user if the workspace cannot be opened with the original cached build state
    652 
    +            if workspace.cached_build and not buildtree:
    653 
    +                self._message(MessageType.WARN, "{} original buildtree artifact not available,"
    654 
    +                              " workspace will be opened with source checkout".format(element.name))
    655 
    +
    656 
    +            workspaces.create_workspace(element._get_full_name(), workspace_path,
    657 
    +                                        cached_build=buildtree)
    658 
    +
    659 
    +            with element.timed_activity("Staging to {}".format(workspace_path)):
    660 
    +                element._open_workspace(buildtree=buildtree)
    621 661
    622 662 
                 self._message(MessageType.INFO,
    623 663 
                               "Reset workspace for {} at: {}".format(element.name,

  • buildstream/_workspaces.py
    ... ... @@ -24,7 +24,7 @@ from . import _yaml
    24 24 
     from ._exceptions import LoadError, LoadErrorReason
    25 25
    26 26
    27 
    -BST_WORKSPACE_FORMAT_VERSION = 3
    27 
    +BST_WORKSPACE_FORMAT_VERSION = 4
    28 28
    29 29
    30 30 
     # Workspace()
    ... ... @@ -43,9 +43,11 @@ BST_WORKSPACE_FORMAT_VERSION = 3
    43 43 
     #    running_files (dict): A dict mapping dependency elements to files
    44 44 
     #                          changed between failed builds. Should be
    45 45 
     #                          made obsolete with failed build artifacts.
    46 
    +#    cached_build (bool): If the workspace is staging the cached build artifact
    46 47 
     #
    47 48 
     class Workspace():
    48 
    -    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
    49 
    +    def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False,
    50 
    +                 running_files=None, cached_build=False):
    49 51 
             self.prepared = prepared
    50 52 
             self.last_successful = last_successful
    51 53 
             self._path = path
    ... ... @@ -53,6 +55,7 @@ class Workspace():
    53 55
    54 56 
             self._toplevel_project = toplevel_project
    55 57 
             self._key = None
    58 
    +        self.cached_build = cached_build
    56 59
    57 60 
         # to_dict()
    58 61 
         #
    ... ... @@ -65,7 +68,8 @@ class Workspace():
    65 68 
             ret = {
    66 69 
                 'prepared': self.prepared,
    67 70 
                 'path': self._path,
    68 
    -            'running_files': self.running_files
    71 
    +            'running_files': self.running_files,
    72 
    +            'cached_build': self.cached_build
    69 73 
             }
    70 74 
             if self.last_successful is not None:
    71 75 
                 ret["last_successful"] = self.last_successful
    ... ... @@ -224,12 +228,13 @@ class Workspaces():
    224 228 
         # Args:
    225 229 
         #    element_name (str) - The element name to create a workspace for
    226 230 
         #    path (str) - The path in which the workspace should be kept
    231 
    +    #    cached_build (bool) - If the workspace is staging the cached build artifact
    227 232 
         #
    228 
    -    def create_workspace(self, element_name, path):
    233 
    +    def create_workspace(self, element_name, path, cached_build=False):
    229 234 
             if path.startswith(self._toplevel_project.directory):
    230 235 
                 path = os.path.relpath(path, self._toplevel_project.directory)
    231 236
    232 
    -        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
    237 
    +        self._workspaces[element_name] = Workspace(self._toplevel_project, path=path, cached_build=cached_build)
    233 238
    234 239 
             return self._workspaces[element_name]
    235 240
    ... ... @@ -396,6 +401,7 @@ class Workspaces():
    396 401 
                 'path': _yaml.node_get(node, str, 'path'),
    397 402 
                 'last_successful': _yaml.node_get(node, str, 'last_successful', default_value=None),
    398 403 
                 'running_files': _yaml.node_get(node, dict, 'running_files', default_value=None),
    404 
    +            'cached_build': _yaml.node_get(node, bool, 'cached_build', default_value=False)
    399 405 
             }
    400 406 
             return Workspace.from_dict(self._toplevel_project, dictionary)
    401 407

  • buildstream/element.py
    ... ... @@ -1880,7 +1880,10 @@ class Element(Plugin):
    1880 1880 
         # This requires that a workspace already be created in
    1881 1881 
         # the workspaces metadata first.
    1882 1882 
         #
    1883 
    -    def _open_workspace(self):
    1883 
    +    # Args:
    1884 
    +    #    buildtree (bool): Whether to open workspace with artifact buildtree
    1885 
    +    #
    1886 
    +    def _open_workspace(self, buildtree=False):
    1884 1887 
             context = self._get_context()
    1885 1888 
             workspace = self._get_workspace()
    1886 1889 
             assert workspace is not None
    ... ... @@ -1893,12 +1896,22 @@ class Element(Plugin):
    1893 1896 
             # files in the target directory actually works without any
    1894 1897 
             # additional support from Source implementations.
    1895 1898 
             #
    1899 
    +
    1896 1900 
             os.makedirs(context.builddir, exist_ok=True)
    1897 
    -        with utils._tempdir(dir=context.builddir, prefix='workspace-{}'
    1898 
    -                            .format(self.normal_name)) as temp:
    1901 
    +        with utils._tempdir(dir=context.builddir, prefix='workspace-source-{}'
    1902 
    +                            .format(self.normal_name)) as temp,\
    1903 
    +            utils._tempdir(dir=context.builddir, prefix='workspace-buildtree-{}'
    1904 
    +                           .format(self.normal_name)) as buildtreetemp:
    1905 
    +
    1899 1906 
                 for source in self.sources():
    1900 1907 
                     source._init_workspace(temp)
    1901 1908
    1909 
    +            # Overwrite the source checkout with the cached buildtree
    1910 
    +            if buildtree:
    1911 
    +                self.__artifacts.checkout_artifact_subdir(self, self._get_cache_key(), 'buildtree', buildtreetemp)
    1912 
    +                if utils._call([utils.get_host_tool('cp'), '-pfr', "".join((buildtreetemp, '/.')), temp])[0] != 0:
    1913 
    +                    raise ElementError("Failed to copy buildtree into workspace checkout at {}".format(buildtreetemp))
    1914 
    +
    1902 1915 
                 # Now hardlink the files into the workspace target.
    1903 1916 
                 utils.link_files(temp, workspace.get_absolute_path())
    1904 1917

  • doc/source/developing/workspaces.rst
    ... ... @@ -24,9 +24,32 @@ Suppose we now want to alter the functionality of the *hello* command. We can
    24 24 
     make changes to the source code of Buildstream elements by making use of
    25 25 
     BuildStream's workspace command.
    26 26
    27 
    +Utilising cached buildtrees
    28 
    +---------------------------
    29 
    + When a BuildStream build element artifact is created and cached, a snapshot of
    30 
    + the build directory after the build commands have completed is included in the
    31 
    + artifact. This `build tree` can be considered an intermediary state of element,
    32 
    + where the source is present along with any output created during the build
    33 
    + execution.
    34 
    +
    35 
    + By default when opening a workspace, bst will attempt to stage the build tree
    36 
    + into the workspace if it's available in the local cache. If the respective
    37 
    + build tree is not present in the cache (element not cached, partially cached or
    38 
    + is a non build element) then the source will be staged as is. The default
    39 
    + behaviour to attempt to use the build tree can be overriden with specific bst
    40 
    + workspace open option of `--no-cache`, or via setting user configuration option
    41 
    + `workspacebuildtrees: False`
    42 
    +
    27 43
    28 44 
     Opening a workspace
    29 45 
     -------------------
    46 
    +.. note::
    47 
    +
    48 
    +    This example presumes you built the hello.bst during
    49 
    +    :ref:`running commands <tutorial_running_commands>`
    50 
    +    if not, please start by building it.
    51 
    +
    52 
    +
    30 53 
     First we need to open a workspace, we can do this by running
    31 54
    32 55 
     .. raw:: html
    ... ... @@ -88,6 +111,15 @@ Alternatively, if we wish to discard the changes we can use
    88 111
    89 112 
     This resets the workspace to its original state.
    90 113
    114 
    +.. note::
    115 
    +
    116 
    +    bst reset will attempt to open the workspace in
    117 
    +    the condition in which it was originally staged,
    118 
    +    i.e with or without consuming the element build tree.
    119 
    +    If it was originally staged with a cached build tree
    120 
    +    and there's no longer one available, the source will
    121 
    +    be staged as is.
    122 
    +
    91 123 
     To discard the workspace completely we can do:
    92 124
    93 125 
     .. raw:: html

  • tests/frontend/workspace.py
    ... ... @@ -44,7 +44,7 @@ DATA_DIR = os.path.join(
    44 44
    45 45
    46 46 
     def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    47 
    -                   project_path=None, element_attrs=None):
    47 
    +                   project_path=None, element_attrs=None, no_cache=False):
    48 48 
         if not workspace_dir:
    49 49 
             workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    50 50 
         if not project_path:
    ... ... @@ -88,6 +88,8 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
    88 88 
         args = ['workspace', 'open']
    89 89 
         if track:
    90 90 
             args.append('--track')
    91 
    +    if no_cache:
    92 
    +        args.append('--no-cache')
    91 93 
         args.extend([element_name, workspace_dir])
    92 94 
         result = cli.run(project=project_path, args=args)
    93 95
    ... ... @@ -101,7 +103,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir
    101 103 
         filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    102 104 
         assert os.path.exists(filename)
    103 105
    104 
    -    return (element_name, project_path, workspace_dir)
    106 
    +    return (element_name, project_path, workspace_dir, result)
    105 107
    106 108
    107 109 
     @pytest.mark.datafiles(DATA_DIR)
    ... ... @@ -112,7 +114,7 @@ def test_open(cli, tmpdir, datafiles, kind):
    112 114
    113 115 
     @pytest.mark.datafiles(DATA_DIR)
    114 116 
     def test_open_bzr_customize(cli, tmpdir, datafiles):
    115 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "bzr", False)
    117 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, "bzr", False)
    116 118
    117 119 
         # Check that the .bzr dir exists
    118 120 
         bzrdir = os.path.join(workspace, ".bzr")
    ... ... @@ -137,7 +139,7 @@ def test_open_track(cli, tmpdir, datafiles, kind):
    137 139 
     @pytest.mark.datafiles(DATA_DIR)
    138 140 
     @pytest.mark.parametrize("kind", repo_kinds)
    139 141 
     def test_open_force(cli, tmpdir, datafiles, kind):
    140 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
    142 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
    141 143
    142 144 
         # Close the workspace
    143 145 
         result = cli.run(project=project, args=[
    ... ... @@ -158,7 +160,7 @@ def test_open_force(cli, tmpdir, datafiles, kind):
    158 160 
     @pytest.mark.datafiles(DATA_DIR)
    159 161 
     @pytest.mark.parametrize("kind", repo_kinds)
    160 162 
     def test_open_force_open(cli, tmpdir, datafiles, kind):
    161 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
    163 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
    162 164
    163 165 
         # Assert the workspace dir exists
    164 166 
         assert os.path.exists(workspace)
    ... ... @@ -173,7 +175,7 @@ def test_open_force_open(cli, tmpdir, datafiles, kind):
    173 175 
     @pytest.mark.datafiles(DATA_DIR)
    174 176 
     @pytest.mark.parametrize("kind", repo_kinds)
    175 177 
     def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
    176 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False, "-alpha")
    178 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False, "-alpha")
    177 179
    178 180 
         # Assert the workspace dir exists
    179 181 
         assert os.path.exists(workspace)
    ... ... @@ -183,7 +185,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
    183 185
    184 186 
         tmpdir = os.path.join(str(tmpdir), "-beta")
    185 187 
         shutil.move(hello_path, hello1_path)
    186 
    -    element_name2, project2, workspace2 = open_workspace(cli, tmpdir, datafiles, kind, False, "-beta")
    188 
    +    element_name2, project2, workspace2, _ = open_workspace(cli, tmpdir, datafiles, kind, False, "-beta")
    187 189
    188 190 
         # Assert the workspace dir exists
    189 191 
         assert os.path.exists(workspace2)
    ... ... @@ -210,7 +212,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
    210 212 
     @pytest.mark.datafiles(DATA_DIR)
    211 213 
     @pytest.mark.parametrize("kind", repo_kinds)
    212 214 
     def test_close(cli, tmpdir, datafiles, kind):
    213 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
    215 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
    214 216
    215 217 
         # Close the workspace
    216 218 
         result = cli.run(project=project, args=[
    ... ... @@ -226,7 +228,7 @@ def test_close(cli, tmpdir, datafiles, kind):
    226 228 
     def test_close_external_after_move_project(cli, tmpdir, datafiles):
    227 229 
         workspace_dir = os.path.join(str(tmpdir), "workspace")
    228 230 
         project_path = os.path.join(str(tmpdir), 'initial_project')
    229 
    -    element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path)
    231 
    +    element_name, _, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path)
    230 232 
         assert os.path.exists(workspace_dir)
    231 233 
         moved_dir = os.path.join(str(tmpdir), 'external_project')
    232 234 
         shutil.move(project_path, moved_dir)
    ... ... @@ -246,8 +248,8 @@ def test_close_external_after_move_project(cli, tmpdir, datafiles):
    246 248 
     def test_close_internal_after_move_project(cli, tmpdir, datafiles):
    247 249 
         initial_dir = os.path.join(str(tmpdir), 'initial_project')
    248 250 
         initial_workspace = os.path.join(initial_dir, 'workspace')
    249 
    -    element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False,
    250 
    -                                        workspace_dir=initial_workspace, project_path=initial_dir)
    251 
    +    element_name, _, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False,
    252 
    +                                           workspace_dir=initial_workspace, project_path=initial_dir)
    251 253 
         moved_dir = os.path.join(str(tmpdir), 'internal_project')
    252 254 
         shutil.move(initial_dir, moved_dir)
    253 255 
         assert os.path.exists(moved_dir)
    ... ... @@ -265,7 +267,7 @@ def test_close_internal_after_move_project(cli, tmpdir, datafiles):
    265 267
    266 268 
     @pytest.mark.datafiles(DATA_DIR)
    267 269 
     def test_close_removed(cli, tmpdir, datafiles):
    268 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    270 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    269 271
    270 272 
         # Remove it first, closing the workspace should work
    271 273 
         shutil.rmtree(workspace)
    ... ... @@ -282,7 +284,7 @@ def test_close_removed(cli, tmpdir, datafiles):
    282 284
    283 285 
     @pytest.mark.datafiles(DATA_DIR)
    284 286 
     def test_close_nonexistant_element(cli, tmpdir, datafiles):
    285 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    287 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    286 288 
         element_path = os.path.join(datafiles.dirname, datafiles.basename, 'elements', element_name)
    287 289
    288 290 
         # First brutally remove the element.bst file, ensuring that
    ... ... @@ -304,9 +306,9 @@ def test_close_nonexistant_element(cli, tmpdir, datafiles):
    304 306 
     def test_close_multiple(cli, tmpdir, datafiles):
    305 307 
         tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
    306 308 
         tmpdir_beta = os.path.join(str(tmpdir), 'beta')
    307 
    -    alpha, project, workspace_alpha = open_workspace(
    309 
    +    alpha, project, workspace_alpha, _ = open_workspace(
    308 310 
             cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
    309 
    -    beta, project, workspace_beta = open_workspace(
    311 
    +    beta, project, workspace_beta, _ = open_workspace(
    310 312 
             cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
    311 313
    312 314 
         # Close the workspaces
    ... ... @@ -324,9 +326,9 @@ def test_close_multiple(cli, tmpdir, datafiles):
    324 326 
     def test_close_all(cli, tmpdir, datafiles):
    325 327 
         tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
    326 328 
         tmpdir_beta = os.path.join(str(tmpdir), 'beta')
    327 
    -    alpha, project, workspace_alpha = open_workspace(
    329 
    +    alpha, project, workspace_alpha, _ = open_workspace(
    328 330 
             cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
    329 
    -    beta, project, workspace_beta = open_workspace(
    331 
    +    beta, project, workspace_beta, _ = open_workspace(
    330 332 
             cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
    331 333
    332 334 
         # Close the workspaces
    ... ... @@ -343,7 +345,7 @@ def test_close_all(cli, tmpdir, datafiles):
    343 345 
     @pytest.mark.datafiles(DATA_DIR)
    344 346 
     def test_reset(cli, tmpdir, datafiles):
    345 347 
         # Open the workspace
    346 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    348 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    347 349
    348 350 
         # Modify workspace
    349 351 
         shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
    ... ... @@ -366,9 +368,9 @@ def test_reset_multiple(cli, tmpdir, datafiles):
    366 368 
         # Open the workspaces
    367 369 
         tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
    368 370 
         tmpdir_beta = os.path.join(str(tmpdir), 'beta')
    369 
    -    alpha, project, workspace_alpha = open_workspace(
    371 
    +    alpha, project, workspace_alpha, _ = open_workspace(
    370 372 
             cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
    371 
    -    beta, project, workspace_beta = open_workspace(
    373 
    +    beta, project, workspace_beta, _ = open_workspace(
    372 374 
             cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
    373 375
    374 376 
         # Modify workspaces
    ... ... @@ -392,9 +394,9 @@ def test_reset_all(cli, tmpdir, datafiles):
    392 394 
         # Open the workspaces
    393 395 
         tmpdir_alpha = os.path.join(str(tmpdir), 'alpha')
    394 396 
         tmpdir_beta = os.path.join(str(tmpdir), 'beta')
    395 
    -    alpha, project, workspace_alpha = open_workspace(
    397 
    +    alpha, project, workspace_alpha, _ = open_workspace(
    396 398 
             cli, tmpdir_alpha, datafiles, 'git', False, suffix='-alpha')
    397 
    -    beta, project, workspace_beta = open_workspace(
    399 
    +    beta, project, workspace_beta, _ = open_workspace(
    398 400 
             cli, tmpdir_beta, datafiles, 'git', False, suffix='-beta')
    399 401
    400 402 
         # Modify workspaces
    ... ... @@ -415,7 +417,7 @@ def test_reset_all(cli, tmpdir, datafiles):
    415 417
    416 418 
     @pytest.mark.datafiles(DATA_DIR)
    417 419 
     def test_list(cli, tmpdir, datafiles):
    418 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    420 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    419 421
    420 422 
         # Now list the workspaces
    421 423 
         result = cli.run(project=project, args=[
    ... ... @@ -437,7 +439,7 @@ def test_list(cli, tmpdir, datafiles):
    437 439 
     @pytest.mark.parametrize("kind", repo_kinds)
    438 440 
     @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
    439 441 
     def test_build(cli, tmpdir, datafiles, kind, strict):
    440 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
    442 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
    441 443 
         checkout = os.path.join(str(tmpdir), 'checkout')
    442 444
    443 445 
         # Modify workspace
    ... ... @@ -516,7 +518,7 @@ def test_buildable_no_ref(cli, tmpdir, datafiles):
    516 518 
     @pytest.mark.parametrize("modification", [("addfile"), ("removefile"), ("modifyfile")])
    517 519 
     @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
    518 520 
     def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
    519 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    521 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    520 522 
         checkout = os.path.join(str(tmpdir), 'checkout')
    521 523
    522 524 
         # Configure strict mode
    ... ... @@ -637,7 +639,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    637 639 
                 "alpha.bst": {
    638 640 
                     "prepared": False,
    639 641 
                     "path": "/workspaces/bravo",
    640 
    -                "running_files": {}
    642 
    +                "running_files": {},
    643 
    +                "cached_build": False
    641 644 
                 }
    642 645 
             }
    643 646 
         }),
    ... ... @@ -652,7 +655,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    652 655 
                 "alpha.bst": {
    653 656 
                     "prepared": False,
    654 657 
                     "path": "/workspaces/bravo",
    655 
    -                "running_files": {}
    658 
    +                "running_files": {},
    659 
    +                "cached_build": False
    656 660 
                 }
    657 661 
             }
    658 662 
         }),
    ... ... @@ -670,7 +674,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    670 674 
                 "alpha.bst": {
    671 675 
                     "prepared": False,
    672 676 
                     "path": "/workspaces/bravo",
    673 
    -                "running_files": {}
    677 
    +                "running_files": {},
    678 
    +                "cached_build": False
    674 679 
                 }
    675 680 
             }
    676 681 
         }),
    ... ... @@ -695,7 +700,8 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    695 700 
                     "last_successful": "some_key",
    696 701 
                     "running_files": {
    697 702 
                         "beta.bst": ["some_file"]
    698 
    -                }
    703 
    +                },
    704 
    +                "cached_build": False
    699 705 
                 }
    700 706 
             }
    701 707 
         }),
    ... ... @@ -715,7 +721,30 @@ def test_list_unsupported_workspace(cli, tmpdir, datafiles, workspace_cfg):
    715 721 
                 "alpha.bst": {
    716 722 
                     "prepared": True,
    717 723 
                     "path": "/workspaces/bravo",
    718 
    -                "running_files": {}
    724 
    +                "running_files": {},
    725 
    +                "cached_build": False
    726 
    +            }
    727 
    +        }
    728 
    +    }),
    729 
    +    # Test loading version 4
    730 
    +    ({
    731 
    +        "format-version": 4,
    732 
    +        "workspaces": {
    733 
    +            "alpha.bst": {
    734 
    +                "prepared": False,
    735 
    +                "path": "/workspaces/bravo",
    736 
    +                "running_files": {},
    737 
    +                "cached_build": True
    738 
    +            }
    739 
    +        }
    740 
    +    }, {
    741 
    +        "format-version": BST_WORKSPACE_FORMAT_VERSION,
    742 
    +        "workspaces": {
    743 
    +            "alpha.bst": {
    744 
    +                "prepared": False,
    745 
    +                "path": "/workspaces/bravo",
    746 
    +                "running_files": {},
    747 
    +                "cached_build": True
    719 748 
                 }
    720 749 
             }
    721 750 
         })
    ... ... @@ -779,7 +808,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
    779 808 
     @pytest.mark.datafiles(DATA_DIR)
    780 809 
     @pytest.mark.parametrize("kind", repo_kinds)
    781 810 
     def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
    782 
    -    element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
    811 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, kind, False)
    783 812
    784 813 
         shutil.rmtree(workspace)
    785 814
    ... ... @@ -793,8 +822,8 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
    793 822 
     @pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
    794 823 
     def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
    795 824 
         checkout = os.path.join(str(tmpdir), 'checkout')
    796 
    -    element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
    797 
    -                                                      datafiles, 'git', False)
    825 
    +    element_name, project, workspace, _ = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
    826 
    +                                                         datafiles, 'git', False)
    798 827
    799 828 
         element_path = os.path.join(project, 'elements')
    800 829 
         back_dep_element_name = 'workspace-test-back-dep.bst'
    ... ... @@ -869,10 +898,76 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles):
    869 898 
                 ]
    870 899 
             }
    871 900 
         }
    872 
    -    element_name, project, _ = open_workspace(cli, tmpdir, datafiles,
    873 
    -                                              "git", False, element_attrs=element_config)
    901 
    +    element_name, project, _, _ = open_workspace(cli, tmpdir, datafiles,
    902 
    +                                                 "git", False, element_attrs=element_config)
    874 903
    875 904 
         for _ in range(2):
    876 905 
             result = cli.run(project=project, args=["build", element_name])
    877 906 
             assert "BUG" not in result.stderr
    878 907 
             assert cli.get_element_state(project, element_name) != "cached"
    908 
    +
    909 
    +
    910 
    +@pytest.mark.datafiles(DATA_DIR)
    911 
    +def test_nocache_open_messages(cli, tmpdir, datafiles):
    912 
    +    # cli default WARN for source dropback possibility when no-cache flag is not passed
    913 
    +    element_name, project, workspace, result = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='1')
    914 
    +    assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" in result.output
    915 
    +
    916 
    +    # cli WARN for source dropback happening when no-cache flag not given, but buildtree not available
    917 
    +    assert "workspace will be opened with source checkout" in result.stderr
    918 
    +
    919 
    +    # cli default WARN for source dropback possibilty not given when no-cache flag is passed
    920 
    +    tmpdir = os.path.join(str(tmpdir), "2")
    921 
    +    element_name, project, workspace, result = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='2',
    922 
    +                                                              no_cache=True)
    923 
    +    assert "WARNING: Workspace will be opened without the cached buildtree if not cached locally" not in result.output
    924 
    +
    925 
    +
    926 
    +@pytest.mark.datafiles(DATA_DIR)
    927 
    +def test_nocache_reset_messages(cli, tmpdir, datafiles):
    928 
    +    element_name, project, workspace, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, suffix='1')
    929 
    +
    930 
    +    # Modify workspace, without building so the artifact is not cached
    931 
    +    shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
    932 
    +    os.makedirs(os.path.join(workspace, 'etc'))
    933 
    +    with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
    934 
    +        f.write("PONY='pink'")
    935 
    +
    936 
    +    # Now reset the open workspace, this should have the
    937 
    +    # effect of reverting our changes to the original source, as it
    938 
    +    # was not originally opened with a cached buildtree and as such
    939 
    +    # should not notify the user
    940 
    +    result = cli.run(project=project, args=[
    941 
    +        'workspace', 'reset', element_name
    942 
    +    ])
    943 
    +    result.assert_success()
    944 
    +    assert "original buildtree artifact not available" not in result.output
    945 
    +    assert os.path.exists(os.path.join(workspace, 'usr', 'bin', 'hello'))
    946 
    +    assert not os.path.exists(os.path.join(workspace, 'etc', 'pony.conf'))
    947 
    +
    948 
    +    # Close the workspace
    949 
    +    result = cli.run(project=project, args=[
    950 
    +        'workspace', 'close', '--remove-dir', element_name
    951 
    +    ])
    952 
    +    result.assert_success()
    953 
    +
    954 
    +    # Build the workspace so we have a cached buildtree artifact for the element
    955 
    +    assert cli.get_element_state(project, element_name) == 'buildable'
    956 
    +    result = cli.run(project=project, args=['build', element_name])
    957 
    +    result.assert_success()
    958 
    +
    959 
    +    # Opening the workspace after a build should lead to the cached buildtree being
    960 
    +    # staged by default
    961 
    +    result = cli.run(project=project, args=[
    962 
    +        'workspace', 'open', element_name, workspace
    963 
    +    ])
    964 
    +    result.assert_success()
    965 
    +    assert "buildtree artifact is available" in result.stderr
    966 
    +
    967 
    +    # Now reset the workspace and ensure that a warning is not given about the artifact
    968 
    +    # buildtree not being available
    969 
    +    result = cli.run(project=project, args=[
    970 
    +        'workspace', 'reset', element_name
    971 
    +    ])
    972 
    +    result.assert_success()
    973 
    +    assert "original buildtree artifact not available" not in result.output

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]