[Notes] [Git][BuildStream/buildstream][relative_workspaces] Implementing relative workspaces



Title: GitLab

Phillip Smyth pushed to branch relative_workspaces at BuildStream / buildstream

Commits:

6 changed files:

Changes:

  • NEWS
    ... ... @@ -17,6 +17,10 @@ buildstream 1.1.5
    17 17
         to not be scheduled and fail during artifact assembly,
    
    18 18
         and display the retry prompt during an interactive session.
    
    19 19
     
    
    20
    +  o Due to enabling the use of relative workspaces, "Legacy" workspaces
    
    21
    +    may need to be closed and remade before the changes will affect them.
    
    22
    +    Downgrading after using this feature may result in workspaces
    
    23
    +    not functioning correctly
    
    20 24
     
    
    21 25
     =================
    
    22 26
     buildstream 1.1.4
    

  • buildstream/_frontend/widget.py
    ... ... @@ -418,7 +418,9 @@ class LogLine(Widget):
    418 418
                 if "%{workspace-dirs" in format_:
    
    419 419
                     workspace = element._get_workspace()
    
    420 420
                     if workspace is not None:
    
    421
    -                    path = workspace.path.replace(os.getenv('HOME', '/root'), '~')
    
    421
    +                    path = workspace.get_absolute_path()
    
    422
    +                    if path.startswith("~/"):
    
    423
    +                        path = os.path.join(os.getenv('HOME', '/root'), path[2:])
    
    422 424
                         line = p.fmt_subst(line, 'workspace-dirs', "Workspace: {}".format(path))
    
    423 425
                     else:
    
    424 426
                         line = p.fmt_subst(
    

  • buildstream/_stream.py
    ... ... @@ -460,7 +460,7 @@ class Stream():
    460 460
                                                   selection=PipelineSelection.REDIRECT,
    
    461 461
                                                   track_selection=PipelineSelection.REDIRECT)
    
    462 462
             target = elements[0]
    
    463
    -        workdir = os.path.abspath(directory)
    
    463
    +        directory = os.path.abspath(directory)
    
    464 464
     
    
    465 465
             if not list(target.sources()):
    
    466 466
                 build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    
    ... ... @@ -476,7 +476,7 @@ class Stream():
    476 476
             workspace = workspaces.get_workspace(target._get_full_name())
    
    477 477
             if workspace and not force:
    
    478 478
                 raise StreamError("Workspace '{}' is already defined at: {}"
    
    479
    -                              .format(target.name, workspace.path))
    
    479
    +                              .format(target.name, workspace.get_absolute_path()))
    
    480 480
     
    
    481 481
             # If we're going to checkout, we need at least a fetch,
    
    482 482
             # if we were asked to track first, we're going to fetch anyway.
    
    ... ... @@ -502,7 +502,7 @@ class Stream():
    502 502
             except OSError as e:
    
    503 503
                 raise StreamError("Failed to create workspace directory: {}".format(e)) from e
    
    504 504
     
    
    505
    -        workspaces.create_workspace(target._get_full_name(), workdir)
    
    505
    +        workspaces.create_workspace(target._get_full_name(), directory)
    
    506 506
     
    
    507 507
             if not no_checkout:
    
    508 508
                 with target.timed_activity("Staging sources to {}".format(directory)):
    
    ... ... @@ -526,12 +526,12 @@ class Stream():
    526 526
             # Remove workspace directory if prompted
    
    527 527
             if remove_dir:
    
    528 528
                 with self._context.timed_activity("Removing workspace directory {}"
    
    529
    -                                              .format(workspace.path)):
    
    529
    +                                              .format(workspace.get_absolute_path())):
    
    530 530
                     try:
    
    531
    -                    shutil.rmtree(workspace.path)
    
    531
    +                    shutil.rmtree(workspace.get_absolute_path())
    
    532 532
                     except OSError as e:
    
    533 533
                         raise StreamError("Could not remove  '{}': {}"
    
    534
    -                                      .format(workspace.path, e)) from e
    
    534
    +                                      .format(workspace.get_absolute_path(), e)) from e
    
    535 535
     
    
    536 536
             # Delete the workspace and save the configuration
    
    537 537
             workspaces.delete_workspace(element_name)
    
    ... ... @@ -574,28 +574,30 @@ class Stream():
    574 574
     
    
    575 575
             for element in elements:
    
    576 576
                 workspace = workspaces.get_workspace(element._get_full_name())
    
    577
    -
    
    577
    +            workspace_path = workspace.get_absolute_path()
    
    578 578
                 if soft:
    
    579 579
                     workspace.prepared = False
    
    580 580
                     self._message(MessageType.INFO, "Reset workspace state for {} at: {}"
    
    581
    -                              .format(element.name, workspace.path))
    
    581
    +                              .format(element.name, workspace_path))
    
    582 582
                     continue
    
    583 583
     
    
    584 584
                 with element.timed_activity("Removing workspace directory {}"
    
    585
    -                                        .format(workspace.path)):
    
    585
    +                                        .format(workspace_path)):
    
    586 586
                     try:
    
    587
    -                    shutil.rmtree(workspace.path)
    
    587
    +                    shutil.rmtree(workspace_path)
    
    588 588
                     except OSError as e:
    
    589 589
                         raise StreamError("Could not remove  '{}': {}"
    
    590
    -                                      .format(workspace.path, e)) from e
    
    590
    +                                      .format(workspace_path, e)) from e
    
    591 591
     
    
    592 592
                 workspaces.delete_workspace(element._get_full_name())
    
    593
    -            workspaces.create_workspace(element._get_full_name(), workspace.path)
    
    593
    +            workspaces.create_workspace(element._get_full_name(), workspace_path)
    
    594 594
     
    
    595
    -            with element.timed_activity("Staging sources to {}".format(workspace.path)):
    
    595
    +            with element.timed_activity("Staging sources to {}".format(workspace_path)):
    
    596 596
                     element._open_workspace()
    
    597 597
     
    
    598
    -            self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, workspace.path))
    
    598
    +            self._message(MessageType.INFO,
    
    599
    +                          "Reset workspace for {} at: {}".format(element.name,
    
    600
    +                                                                 workspace_path))
    
    599 601
     
    
    600 602
             workspaces.save_config()
    
    601 603
     
    
    ... ... @@ -632,7 +634,7 @@ class Stream():
    632 634
             for element_name, workspace_ in self._context.get_workspaces().list():
    
    633 635
                 workspace_detail = {
    
    634 636
                     'element': element_name,
    
    635
    -                'directory': workspace_.path,
    
    637
    +                'directory': workspace_.get_absolute_path(),
    
    636 638
                 }
    
    637 639
                 workspaces.append(workspace_detail)
    
    638 640
     
    

  • buildstream/_workspaces.py
    ... ... @@ -26,14 +26,6 @@ from ._exceptions import LoadError, LoadErrorReason
    26 26
     
    
    27 27
     BST_WORKSPACE_FORMAT_VERSION = 3
    
    28 28
     
    
    29
    -# Hold on to a list of members which get serialized
    
    30
    -_WORKSPACE_MEMBERS = [
    
    31
    -    'prepared',
    
    32
    -    'path',
    
    33
    -    'last_successful',
    
    34
    -    'running_files'
    
    35
    -]
    
    36
    -
    
    37 29
     
    
    38 30
     # Workspace()
    
    39 31
     #
    
    ... ... @@ -56,7 +48,7 @@ class Workspace():
    56 48
         def __init__(self, toplevel_project, *, last_successful=None, path=None, prepared=False, running_files=None):
    
    57 49
             self.prepared = prepared
    
    58 50
             self.last_successful = last_successful
    
    59
    -        self.path = path
    
    51
    +        self._path = path
    
    60 52
             self.running_files = running_files if running_files is not None else {}
    
    61 53
     
    
    62 54
             self._toplevel_project = toplevel_project
    
    ... ... @@ -64,14 +56,20 @@ class Workspace():
    64 56
     
    
    65 57
         # to_dict()
    
    66 58
         #
    
    67
    -    # Convert this object to a dict for serialization purposes
    
    59
    +    # Convert a list of members which get serialized to a dict for serialization purposes
    
    68 60
         #
    
    69 61
         # Returns:
    
    70 62
         #     (dict) A dict representation of the workspace
    
    71 63
         #
    
    72 64
         def to_dict(self):
    
    73
    -        return {key: val for key, val in self.__dict__.items()
    
    74
    -                if key in _WORKSPACE_MEMBERS and val is not None}
    
    65
    +        ret = {
    
    66
    +            'prepared': self.prepared,
    
    67
    +            'path': self._path,
    
    68
    +            'running_files': self.running_files
    
    69
    +        }
    
    70
    +        if self.last_successful is not None:
    
    71
    +            ret["last_successful"] = self.last_successful
    
    72
    +        return ret
    
    75 73
     
    
    76 74
         # from_dict():
    
    77 75
         #
    
    ... ... @@ -103,15 +101,7 @@ class Workspace():
    103 101
         #    True if the workspace differs from 'other', otherwise False
    
    104 102
         #
    
    105 103
         def differs(self, other):
    
    106
    -
    
    107
    -        for member in _WORKSPACE_MEMBERS:
    
    108
    -            member_a = getattr(self, member)
    
    109
    -            member_b = getattr(other, member)
    
    110
    -
    
    111
    -            if member_a != member_b:
    
    112
    -                return True
    
    113
    -
    
    114
    -        return False
    
    104
    +        return self.to_dict() != other.to_dict()
    
    115 105
     
    
    116 106
         # invalidate_key()
    
    117 107
         #
    
    ... ... @@ -133,7 +123,7 @@ class Workspace():
    133 123
             if os.path.isdir(fullpath):
    
    134 124
                 utils.copy_files(fullpath, directory)
    
    135 125
             else:
    
    136
    -            destfile = os.path.join(directory, os.path.basename(self.path))
    
    126
    +            destfile = os.path.join(directory, os.path.basename(self.get_absolute_path()))
    
    137 127
                 utils.safe_copy(fullpath, destfile)
    
    138 128
     
    
    139 129
         # add_running_files()
    
    ... ... @@ -189,7 +179,7 @@ class Workspace():
    189 179
                     filelist = utils.list_relative_paths(fullpath)
    
    190 180
                     filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist]
    
    191 181
                 else:
    
    192
    -                filelist = [(self.path, fullpath)]
    
    182
    +                filelist = [(self.get_absolute_path(), fullpath)]
    
    193 183
     
    
    194 184
                 self._key = [(relpath, unique_key(fullpath)) for relpath, fullpath in filelist]
    
    195 185
     
    
    ... ... @@ -200,7 +190,7 @@ class Workspace():
    200 190
         # Returns: The absolute path of the element's workspace.
    
    201 191
         #
    
    202 192
         def get_absolute_path(self):
    
    203
    -        return os.path.join(self._toplevel_project.directory, self.path)
    
    193
    +        return os.path.join(self._toplevel_project.directory, self._path)
    
    204 194
     
    
    205 195
     
    
    206 196
     # Workspaces()
    
    ... ... @@ -236,6 +226,9 @@ class Workspaces():
    236 226
         #    path (str) - The path in which the workspace should be kept
    
    237 227
         #
    
    238 228
         def create_workspace(self, element_name, path):
    
    229
    +        if path.startswith(self._toplevel_project.directory):
    
    230
    +            path = os.path.relpath(path, self._toplevel_project.directory)
    
    231
    +
    
    239 232
             self._workspaces[element_name] = Workspace(self._toplevel_project, path=path)
    
    240 233
     
    
    241 234
             return self._workspaces[element_name]
    

  • buildstream/element.py
    ... ... @@ -1403,7 +1403,8 @@ class Element(Plugin):
    1403 1403
                         # If mount_workspaces is set and we're doing incremental builds,
    
    1404 1404
                         # the workspace is already mounted into the sandbox.
    
    1405 1405
                         if not (mount_workspaces and self.__can_build_incrementally()):
    
    1406
    -                        with self.timed_activity("Staging local files at {}".format(workspace.path)):
    
    1406
    +                        with self.timed_activity("Staging local files at {}"
    
    1407
    +                                                 .format(workspace.get_absolute_path())):
    
    1407 1408
                                 workspace.stage(temp_staging_directory)
    
    1408 1409
                     else:
    
    1409 1410
                         # No workspace, stage directly
    
    ... ... @@ -1411,6 +1412,7 @@ class Element(Plugin):
    1411 1412
                             source._stage(temp_staging_directory)
    
    1412 1413
     
    
    1413 1414
                     vdirectory.import_files(temp_staging_directory)
    
    1415
    +
    
    1414 1416
             # Ensure deterministic mtime of sources at build time
    
    1415 1417
             vdirectory.set_deterministic_mtime()
    
    1416 1418
             # Ensure deterministic owners of sources at build time
    
    ... ... @@ -1566,7 +1568,7 @@ class Element(Plugin):
    1566 1568
                             path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep)
    
    1567 1569
                             sandbox_vpath = sandbox_vroot.descend(path_components)
    
    1568 1570
                             try:
    
    1569
    -                            sandbox_vpath.import_files(workspace.path)
    
    1571
    +                            sandbox_vpath.import_files(workspace.get_absolute_path())
    
    1570 1572
                             except UtilError as e:
    
    1571 1573
                                 self.warn("Failed to preserve workspace state for failed build sysroot: {}"
    
    1572 1574
                                           .format(e))
    
    ... ... @@ -1893,7 +1895,7 @@ class Element(Plugin):
    1893 1895
                     source._init_workspace(temp)
    
    1894 1896
     
    
    1895 1897
                 # Now hardlink the files into the workspace target.
    
    1896
    -            utils.link_files(temp, workspace.path)
    
    1898
    +            utils.link_files(temp, workspace.get_absolute_path())
    
    1897 1899
     
    
    1898 1900
         # _get_workspace():
    
    1899 1901
         #
    

  • tests/frontend/workspace.py
    ... ... @@ -18,12 +18,13 @@ DATA_DIR = os.path.join(
    18 18
     )
    
    19 19
     
    
    20 20
     
    
    21
    -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
    
    22
    -    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    23
    -    bin_files_path = os.path.join(project, 'files', 'bin-files')
    
    24
    -    element_path = os.path.join(project, 'elements')
    
    21
    +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None):
    
    22
    +    if not workspace_dir:
    
    23
    +        workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    
    24
    +    project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    25
    +    bin_files_path = os.path.join(project_path, 'files', 'bin-files')
    
    26
    +    element_path = os.path.join(project_path, 'elements')
    
    25 27
         element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
    
    26
    -    workspace = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    
    27 28
     
    
    28 29
         # Create our repo object of the given source type with
    
    29 30
         # the bin files, and then collect the initial ref.
    
    ... ... @@ -45,7 +46,7 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
    45 46
                                 element_name))
    
    46 47
     
    
    47 48
         # Assert that there is no reference, a track & fetch is needed
    
    48
    -    state = cli.get_element_state(project, element_name)
    
    49
    +    state = cli.get_element_state(project_path, element_name)
    
    49 50
         if track:
    
    50 51
             assert state == 'no reference'
    
    51 52
         else:
    
    ... ... @@ -56,20 +57,20 @@ def open_workspace(cli, tmpdir, datafiles, kind, track, suffix=''):
    56 57
         args = ['workspace', 'open']
    
    57 58
         if track:
    
    58 59
             args.append('--track')
    
    59
    -    args.extend([element_name, workspace])
    
    60
    +    args.extend([element_name, workspace_dir])
    
    61
    +    result = cli.run(project=project_path, args=args)
    
    60 62
     
    
    61
    -    result = cli.run(project=project, args=args)
    
    62 63
         result.assert_success()
    
    63 64
     
    
    64 65
         # Assert that we are now buildable because the source is
    
    65 66
         # now cached.
    
    66
    -    assert cli.get_element_state(project, element_name) == 'buildable'
    
    67
    +    assert cli.get_element_state(project_path, element_name) == 'buildable'
    
    67 68
     
    
    68 69
         # Check that the executable hello file is found in the workspace
    
    69
    -    filename = os.path.join(workspace, 'usr', 'bin', 'hello')
    
    70
    +    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    
    70 71
         assert os.path.exists(filename)
    
    71 72
     
    
    72
    -    return (element_name, project, workspace)
    
    73
    +    return (element_name, project_path, workspace_dir)
    
    73 74
     
    
    74 75
     
    
    75 76
     @pytest.mark.datafiles(DATA_DIR)
    
    ... ... @@ -190,6 +191,45 @@ def test_close(cli, tmpdir, datafiles, kind):
    190 191
         assert not os.path.exists(workspace)
    
    191 192
     
    
    192 193
     
    
    194
    +@pytest.mark.datafiles(DATA_DIR)
    
    195
    +def test_close_external_after_move_project(cli, tmpdir, datafiles):
    
    196
    +    tmp_parent = os.path.dirname(str(tmpdir))
    
    197
    +    workspace_dir = os.path.join(tmp_parent, "workspace")
    
    198
    +    element_name, project_path, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir)
    
    199
    +    assert os.path.exists(workspace_dir)
    
    200
    +    tmp_dir = os.path.join(tmp_parent, 'external_project')
    
    201
    +    shutil.move(project_path, tmp_dir)
    
    202
    +    assert os.path.exists(tmp_dir)
    
    203
    +
    
    204
    +    # Close the workspace
    
    205
    +    result = cli.run(configure=False, project=tmp_dir, args=[
    
    206
    +        'workspace', 'close', '--remove-dir', element_name
    
    207
    +    ])
    
    208
    +    result.assert_success()
    
    209
    +
    
    210
    +    # Assert the workspace dir has been deleted
    
    211
    +    assert not os.path.exists(workspace_dir)
    
    212
    +    shutil.move(tmp_dir, project_path)
    
    213
    +
    
    214
    +
    
    215
    +@pytest.mark.datafiles(DATA_DIR)
    
    216
    +def test_close_internal_after_move_project(cli, tmpdir, datafiles):
    
    217
    +    element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
    
    218
    +    tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_project')
    
    219
    +    shutil.move(str(tmpdir), tmp_dir)
    
    220
    +    assert os.path.exists(tmp_dir)
    
    221
    +
    
    222
    +    # Close the workspace
    
    223
    +    result = cli.run(configure=False, project=tmp_dir, args=[
    
    224
    +        'workspace', 'close', '--remove-dir', element_name
    
    225
    +    ])
    
    226
    +    result.assert_success()
    
    227
    +
    
    228
    +    # Assert the workspace dir has been deleted
    
    229
    +    workspace = os.path.join(tmp_dir, 'workspace')
    
    230
    +    assert not os.path.exists(workspace)
    
    231
    +
    
    232
    +
    
    193 233
     @pytest.mark.datafiles(DATA_DIR)
    
    194 234
     def test_close_removed(cli, tmpdir, datafiles):
    
    195 235
         element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, 'git', False)
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]