[Notes] [Git][BuildStream/buildstream][willsalmon/defaultWorkspaces] 9 commits: plugins/sources/pip.py: Accomodate characters '-', '.', '_' for packages



Title: GitLab

Will Salmon pushed to branch willsalmon/defaultWorkspaces at BuildStream / buildstream

Commits:

24 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -166,6 +166,12 @@ docs:
    166 166
         BST_EXT_REF: 1d6ab71151b93c8cbc0a91a36ffe9270f3b835f1 # 0.5.1
    
    167 167
         FD_SDK_REF: 88d7c22c2281b987faa02edd57df80d430eecf1f # 18.08.11-35-g88d7c22c
    
    168 168
       before_script:
    
    169
    +  - |
    
    170
    +    mkdir -p "${HOME}/.config"
    
    171
    +    cat <<EOF >"${HOME}/.config/buildstream.conf"
    
    172
    +    scheduler:
    
    173
    +      fetchers: 2
    
    174
    +    EOF
    
    169 175
       - (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
    
    170 176
       - pip3 install --user -e ${BST_EXT_URL}@${BST_EXT_REF}#egg=bst_ext
    
    171 177
       - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
    

  • buildstream/_context.py
    ... ... @@ -60,6 +60,9 @@ class Context():
    60 60
             # The directory where build sandboxes will be created
    
    61 61
             self.builddir = None
    
    62 62
     
    
    63
    +        # Default root location for workspaces
    
    64
    +        self.workspacedir = None
    
    65
    +
    
    63 66
             # The local binary artifact cache directory
    
    64 67
             self.artifactdir = None
    
    65 68
     
    
    ... ... @@ -161,10 +164,10 @@ class Context():
    161 164
             _yaml.node_validate(defaults, [
    
    162 165
                 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    
    163 166
                 'scheduler', 'artifacts', 'logging', 'projects',
    
    164
    -            'cache'
    
    167
    +            'cache', 'workspacedir',
    
    165 168
             ])
    
    166 169
     
    
    167
    -        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
    
    170
    +        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
    
    168 171
                 # Allow the ~ tilde expansion and any environment variables in
    
    169 172
                 # path specification in the config files.
    
    170 173
                 #
    

  • buildstream/_frontend/cli.py
    ... ... @@ -6,6 +6,7 @@ from .. import _yaml
    6 6
     from .._exceptions import BstError, LoadError, AppError
    
    7 7
     from .._versions import BST_FORMAT_VERSION
    
    8 8
     from .complete import main_bashcomplete, complete_path, CompleteUnhandled
    
    9
    +from ..utils import DirectoryDescription
    
    9 10
     
    
    10 11
     
    
    11 12
     ##################################################################
    
    ... ... @@ -678,28 +679,36 @@ def workspace():
    678 679
     @click.option('--no-checkout', default=False, is_flag=True,
    
    679 680
                   help="Do not checkout the source, only link to the given directory")
    
    680 681
     @click.option('--force', '-f', default=False, is_flag=True,
    
    681
    -              help="Overwrite files existing in checkout directory")
    
    682
    +              help="The workspace will be created even if the directory in which it will be created is not empty " +
    
    683
    +              "or if a workspace for that element already exists")
    
    682 684
     @click.option('--track', 'track_', default=False, is_flag=True,
    
    683 685
                   help="Track and fetch new source references before checking out the workspace")
    
    684
    -@click.argument('element',
    
    685
    -                type=click.Path(readable=False))
    
    686
    -@click.argument('directory', type=click.Path(file_okay=False))
    
    686
    +@click.option('--directory', type=click.Path(file_okay=False), default=None,
    
    687
    +              help="Only for use when a single Element is given: Set the directory to use to create the workspace")
    
    688
    +@click.argument('elements', nargs=-1, type=click.Path(readable=False))
    
    687 689
     @click.pass_obj
    
    688
    -def workspace_open(app, no_checkout, force, track_, element, directory):
    
    690
    +def workspace_open(app, no_checkout, force, track_, directory, elements):
    
    689 691
         """Open a workspace for manual source modification"""
    
    690
    -
    
    691
    -    if os.path.exists(directory):
    
    692
    -
    
    693
    -        if not os.path.isdir(directory):
    
    694
    -            click.echo("Checkout directory is not a directory: {}".format(directory), err=True)
    
    692
    +    directories = []
    
    693
    +    if directory is not None:
    
    694
    +        if len(elements) > 1:
    
    695
    +            click.echo("Directory option can only be used if a single element is given", err=True)
    
    695 696
                 sys.exit(-1)
    
    697
    +        if os.path.exists(directory):
    
    698
    +            if not os.path.isdir(directory):
    
    699
    +                click.echo("Directory path is not a directory: {}".format(directory), err=True)
    
    700
    +                sys.exit(-1)
    
    696 701
     
    
    697
    -        if not (no_checkout or force) and os.listdir(directory):
    
    698
    -            click.echo("Checkout directory is not empty: {}".format(directory), err=True)
    
    699
    -            sys.exit(-1)
    
    702
    +            if not (no_checkout or force) and os.listdir(directory):
    
    703
    +                click.echo("Directory path is not empty: {}".format(directory), err=True)
    
    704
    +                sys.exit(-1)
    
    705
    +        directories.append(DirectoryDescription(directory, use_default=False))
    
    706
    +    else:
    
    707
    +        for element in elements:
    
    708
    +            directories.append(DirectoryDescription(element.rstrip('.bst')))
    
    700 709
     
    
    701 710
         with app.initialized():
    
    702
    -        app.stream.workspace_open(element, directory,
    
    711
    +        app.stream.workspace_open(elements, directories,
    
    703 712
                                       no_checkout=no_checkout,
    
    704 713
                                       track_first=track_,
    
    705 714
                                       force=force)
    

  • buildstream/_stream.py
    ... ... @@ -36,6 +36,7 @@ from . import utils, _yaml, _site
    36 36
     from . import Scope, Consistency
    
    37 37
     
    
    38 38
     
    
    39
    +
    
    39 40
     # Stream()
    
    40 41
     #
    
    41 42
     # This is the main, toplevel calling interface in BuildStream core.
    
    ... ... @@ -448,44 +449,29 @@ class Stream():
    448 449
         # Open a project workspace
    
    449 450
         #
    
    450 451
         # Args:
    
    451
    -    #    target (str): The target element to open the workspace for
    
    452
    -    #    directory (str): The directory to stage the source in
    
    452
    +    #    target (list): List of target elements to open workspaces for
    
    453
    +    #    directory (list): List of DirectoryDescription objects to stage the source in
    
    453 454
         #    no_checkout (bool): Whether to skip checking out the source
    
    454 455
         #    track_first (bool): Whether to track and fetch first
    
    455 456
         #    force (bool): Whether to ignore contents in an existing directory
    
    456 457
         #
    
    457
    -    def workspace_open(self, target, directory, *,
    
    458
    +    def workspace_open(self, targets, directories, *,
    
    458 459
                            no_checkout,
    
    459 460
                            track_first,
    
    460 461
                            force):
    
    462
    +        # This function is a little funny but it is trying to be as atomic as possible.
    
    461 463
     
    
    462 464
             if track_first:
    
    463
    -            track_targets = (target,)
    
    465
    +            track_targets = targets
    
    464 466
             else:
    
    465 467
                 track_targets = ()
    
    466 468
     
    
    467
    -        elements, track_elements = self._load((target,), track_targets,
    
    469
    +        elements, track_elements = self._load(targets, track_targets,
    
    468 470
                                                   selection=PipelineSelection.REDIRECT,
    
    469 471
                                                   track_selection=PipelineSelection.REDIRECT)
    
    470
    -        target = elements[0]
    
    471
    -        directory = os.path.abspath(directory)
    
    472
    -
    
    473
    -        if not list(target.sources()):
    
    474
    -            build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    
    475
    -            if not build_depends:
    
    476
    -                raise StreamError("The given element has no sources")
    
    477
    -            detail = "Try opening a workspace on one of its dependencies instead:\n"
    
    478
    -            detail += "  \n".join(build_depends)
    
    479
    -            raise StreamError("The given element has no sources", detail=detail)
    
    480 472
     
    
    481 473
             workspaces = self._context.get_workspaces()
    
    482 474
     
    
    483
    -        # Check for workspace config
    
    484
    -        workspace = workspaces.get_workspace(target._get_full_name())
    
    485
    -        if workspace and not force:
    
    486
    -            raise StreamError("Workspace '{}' is already defined at: {}"
    
    487
    -                              .format(target.name, workspace.get_absolute_path()))
    
    488
    -
    
    489 475
             # If we're going to checkout, we need at least a fetch,
    
    490 476
             # if we were asked to track first, we're going to fetch anyway.
    
    491 477
             #
    
    ... ... @@ -495,29 +481,69 @@ class Stream():
    495 481
                     track_elements = elements
    
    496 482
                 self._fetch(elements, track_elements=track_elements)
    
    497 483
     
    
    498
    -        if not no_checkout and target._get_consistency() != Consistency.CACHED:
    
    499
    -            raise StreamError("Could not stage uncached source. " +
    
    500
    -                              "Use `--track` to track and " +
    
    501
    -                              "fetch the latest version of the " +
    
    502
    -                              "source.")
    
    484
    +        expanded_directories = []
    
    485
    +        #  To try to be more atomic, loop through the elements and raise any errors we can early
    
    486
    +        for target, directory_obj in zip(elements, directories):
    
    487
    +
    
    488
    +            if not list(target.sources()):
    
    489
    +                build_depends = [x.name for x in target.dependencies(Scope.BUILD, recurse=False)]
    
    490
    +                if not build_depends:
    
    491
    +                    raise StreamError("The element {}  has no sources".format(target.name))
    
    492
    +                detail = "Try opening a workspace on one of its dependencies instead:\n"
    
    493
    +                detail += "  \n".join(build_depends)
    
    494
    +                raise StreamError("The element {} has no sources".format(target.name), detail=detail)
    
    495
    +
    
    496
    +            # Check for workspace config
    
    497
    +            workspace = workspaces.get_workspace(target._get_full_name())
    
    498
    +            if workspace and not force:
    
    499
    +                raise StreamError("Workspace '{}' is already defined at: {}"
    
    500
    +                                  .format(target.name, workspace.get_absolute_path()))
    
    501
    +
    
    502
    +            if not no_checkout and target._get_consistency() != Consistency.CACHED:
    
    503
    +                raise StreamError("Could not stage uncached source. For {} ".format(target.name) +
    
    504
    +                                  "Use `--track` to track and " +
    
    505
    +                                  "fetch the latest version of the " +
    
    506
    +                                  "source.")
    
    507
    +
    
    508
    +            if directory_obj.use_default:
    
    509
    +                directory = os.path.abspath(os.path.join(self._context.workspacedir, directory_obj.directory))
    
    510
    +            else:
    
    511
    +                directory = directory_obj.directory
    
    503 512
     
    
    504
    -        if workspace:
    
    505
    -            workspaces.delete_workspace(target._get_full_name())
    
    506
    -            workspaces.save_config()
    
    507
    -            shutil.rmtree(directory)
    
    508
    -        try:
    
    509
    -            os.makedirs(directory, exist_ok=True)
    
    510
    -        except OSError as e:
    
    511
    -            raise StreamError("Failed to create workspace directory: {}".format(e)) from e
    
    513
    +            expanded_directories.append(directory)
    
    512 514
     
    
    513
    -        workspaces.create_workspace(target._get_full_name(), directory)
    
    515
    +        # So far this function has tried to catch as many issues as possible with out making any changes
    
    516
    +        # Now it dose the bits that can not be made atomic.
    
    517
    +        targetGenerator = zip(elements, expanded_directories)
    
    518
    +        for target, directory in targetGenerator:
    
    519
    +            self._message(MessageType.INFO, "Creating workspace for element {}"
    
    520
    +                          .format(target.name))
    
    514 521
     
    
    515
    -        if not no_checkout:
    
    516
    -            with target.timed_activity("Staging sources to {}".format(directory)):
    
    517
    -                target._open_workspace()
    
    522
    +            workspace = workspaces.get_workspace(target._get_full_name())
    
    523
    +            if workspace:
    
    524
    +                workspaces.delete_workspace(target._get_full_name())
    
    525
    +                workspaces.save_config()
    
    526
    +                shutil.rmtree(directory)
    
    527
    +            try:
    
    528
    +                os.makedirs(directory, exist_ok=True)
    
    529
    +            except OSError as e:
    
    530
    +                todo_elements = " ".join([str(target.name) for target, directory_dict in targetGenerator])
    
    531
    +                if todo_elements:
    
    532
    +                    # This output should make creating the remaining workspaces as easy as possible.
    
    533
    +                    todo_elements = "\nDid not try to create workspaces for " + todo_elements
    
    534
    +                raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e
    
    518 535
     
    
    519
    -        workspaces.save_config()
    
    520
    -        self._message(MessageType.INFO, "Saved workspace configuration")
    
    536
    +            workspaces.create_workspace(target._get_full_name(), directory)
    
    537
    +
    
    538
    +            if not no_checkout:
    
    539
    +                with target.timed_activity("Staging sources to {}".format(directory)):
    
    540
    +                    target._open_workspace()
    
    541
    +
    
    542
    +            # Saving the workspace once it is set up means that if the next workspace fails to be created before
    
    543
    +            # the configuration gets saved. The successfully created workspace still gets saved.
    
    544
    +            workspaces.save_config()
    
    545
    +            self._message(MessageType.INFO, "Added element {} to the workspace configuration"
    
    546
    +                          .format(target._get_full_name()))
    
    521 547
     
    
    522 548
         # workspace_close
    
    523 549
         #
    

  • buildstream/data/userconfig.yaml
    ... ... @@ -22,6 +22,9 @@ artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
    22 22
     # Location to store build logs
    
    23 23
     logdir: ${XDG_CACHE_HOME}/buildstream/logs
    
    24 24
     
    
    25
    +# Default root location for workspaces, blank for no default set.
    
    26
    +workspacedir: .
    
    27
    +
    
    25 28
     #
    
    26 29
     #    Cache
    
    27 30
     #
    

  • buildstream/element.py
    ... ... @@ -2180,6 +2180,7 @@ class Element(Plugin):
    2180 2180
                                         stderr=stderr,
    
    2181 2181
                                         config=config,
    
    2182 2182
                                         server_url=self.__remote_execution_url,
    
    2183
    +                                    bare_directory=bare_directory,
    
    2183 2184
                                         allow_real_directory=False)
    
    2184 2185
                 yield sandbox
    
    2185 2186
     
    

  • buildstream/plugins/sources/pip.py
    ... ... @@ -96,7 +96,7 @@ _PYTHON_VERSIONS = [
    96 96
     # Names of source distribution archives must be of the form
    
    97 97
     # '%{package-name}-%{version}.%{extension}'.
    
    98 98
     _SDIST_RE = re.compile(
    
    99
    -    r'^([a-zA-Z0-9]+?)-(.+).(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    
    99
    +    r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
    
    100 100
         re.IGNORECASE)
    
    101 101
     
    
    102 102
     
    
    ... ... @@ -225,12 +225,27 @@ class PipSource(Source):
    225 225
         def _parse_sdist_names(self, basedir):
    
    226 226
             reqs = []
    
    227 227
             for f in os.listdir(basedir):
    
    228
    -            pkg_match = _SDIST_RE.match(f)
    
    229
    -            if pkg_match:
    
    230
    -                reqs.append(pkg_match.groups())
    
    228
    +            pkg = _match_package_name(f)
    
    229
    +            if pkg is not None:
    
    230
    +                reqs.append(pkg)
    
    231 231
     
    
    232 232
             return sorted(reqs)
    
    233 233
     
    
    234 234
     
    
    235
    +# Extract the package name and version of a source distribution
    
    236
    +#
    
    237
    +# Args:
    
    238
    +#    filename (str): Filename of the source distribution
    
    239
    +#
    
    240
    +# Returns:
    
    241
    +#    (tuple): A tuple of (package_name, version)
    
    242
    +#
    
    243
    +def _match_package_name(filename):
    
    244
    +    pkg_match = _SDIST_RE.match(filename)
    
    245
    +    if pkg_match is None:
    
    246
    +        return None
    
    247
    +    return pkg_match.groups()
    
    248
    +
    
    249
    +
    
    235 250
     def setup():
    
    236 251
         return PipSource

  • buildstream/utils.py
    ... ... @@ -105,6 +105,20 @@ class FileListResult():
    105 105
             return ret
    
    106 106
     
    
    107 107
     
    
    108
    +class DirectoryDescription():
    
    109
    +    """
    
    110
    +    This object is used to keep information about directories in a nice tidy object.
    
    111
    +    """
    
    112
    +    def __init__(self, directory, *, use_default=True):
    
    113
    +        """
    
    114
    +        Args:
    
    115
    +            directory (str): The path to the directory this object describes
    
    116
    +            use_default (bool): Weather to process the directory so it is in the default folder.
    
    117
    +        """
    
    118
    +        self.directory = directory
    
    119
    +        self.use_default = use_default
    
    120
    +
    
    121
    +
    
    108 122
     def list_relative_paths(directory, *, list_dirs=True):
    
    109 123
         """A generator for walking directory relative paths
    
    110 124
     
    
    ... ... @@ -559,6 +573,19 @@ def save_file_atomic(filename, mode='w', *, buffering=-1, encoding=None,
    559 573
             raise
    
    560 574
     
    
    561 575
     
    
    576
    +# DirectoryDescription()
    
    577
    +#
    
    578
    +# This object is used to keep information about directories in a nice tidy object.
    
    579
    +#
    
    580
    +# Args:
    
    581
    +#    directory (str): The path to the directory this object describes
    
    582
    +#    use_default (bool): Weather to process the directory so it is in the default folder.
    
    583
    +#
    
    584
    +class DirectoryDescription():
    
    585
    +    def __init__(self, directory, *, use_default=True):
    
    586
    +        self.directory = directory
    
    587
    +        self.use_default = use_default
    
    588
    +
    
    562 589
     # _get_dir_size():
    
    563 590
     #
    
    564 591
     # Get the disk usage of a given directory in bytes.
    

  • tests/examples/developing.py
    ... ... @@ -55,7 +55,7 @@ def test_open_workspace(cli, tmpdir, datafiles):
    55 55
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    56 56
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
    
    57 57
     
    
    58
    -    result = cli.run(project=project, args=['workspace', 'open', '-f', 'hello.bst', workspace_dir])
    
    58
    +    result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst', ])
    
    59 59
         result.assert_success()
    
    60 60
     
    
    61 61
         result = cli.run(project=project, args=['workspace', 'list'])
    
    ... ... @@ -72,7 +72,7 @@ def test_make_change_in_workspace(cli, tmpdir, datafiles):
    72 72
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    73 73
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello")
    
    74 74
     
    
    75
    -    result = cli.run(project=project, args=['workspace', 'open', '-f', 'hello.bst', workspace_dir])
    
    75
    +    result = cli.run(project=project, args=['workspace', 'open', '-f', '--directory', workspace_dir, 'hello.bst'])
    
    76 76
         result.assert_success()
    
    77 77
     
    
    78 78
         result = cli.run(project=project, args=['workspace', 'list'])
    

  • tests/examples/junctions.py
    ... ... @@ -44,7 +44,7 @@ def test_open_cross_junction_workspace(cli, tmpdir, datafiles):
    44 44
         workspace_dir = os.path.join(str(tmpdir), "workspace_hello_junction")
    
    45 45
     
    
    46 46
         result = cli.run(project=project,
    
    47
    -                     args=['workspace', 'open', 'hello-junction.bst:hello.bst', workspace_dir])
    
    47
    +                     args=['workspace', 'open', '--directory', workspace_dir, 'hello-junction.bst:hello.bst'])
    
    48 48
         result.assert_success()
    
    49 49
     
    
    50 50
         result = cli.run(project=project,
    

  • tests/frontend/buildcheckout.py
    ... ... @@ -509,7 +509,7 @@ def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles):
    509 509
     
    
    510 510
         # Now open a workspace on the junction
    
    511 511
         #
    
    512
    -    result = cli.run(project=project, args=['workspace', 'open', 'junction.bst', workspace])
    
    512
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, 'junction.bst'])
    
    513 513
         result.assert_success()
    
    514 514
         filename = os.path.join(workspace, 'files', 'etc-files', 'etc', 'animal.conf')
    
    515 515
     
    

  • tests/frontend/cross_junction_workspace.py
    ... ... @@ -47,7 +47,7 @@ def open_cross_junction(cli, tmpdir):
    47 47
         workspace = tmpdir.join("workspace")
    
    48 48
     
    
    49 49
         element = 'sub.bst:data.bst'
    
    50
    -    args = ['workspace', 'open', element, str(workspace)]
    
    50
    +    args = ['workspace', 'open', '--directory', str(workspace), element]
    
    51 51
         result = cli.run(project=project, args=args)
    
    52 52
         result.assert_success()
    
    53 53
     
    

  • tests/frontend/workspace.py
    ... ... @@ -21,9 +21,11 @@
    21 21
     #           Phillip Smyth <phillip smyth codethink co uk>
    
    22 22
     #           Jonathan Maw <jonathan maw codethink co uk>
    
    23 23
     #           Richard Maw <richard maw codethink co uk>
    
    24
    +#           William Salmon <will salmon codethink co uk>
    
    24 25
     #
    
    25 26
     
    
    26 27
     import os
    
    28
    +import stat
    
    27 29
     import pytest
    
    28 30
     import shutil
    
    29 31
     import subprocess
    
    ... ... @@ -43,65 +45,118 @@ DATA_DIR = os.path.join(
    43 45
     )
    
    44 46
     
    
    45 47
     
    
    46
    -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    
    47
    -                   project_path=None, element_attrs=None):
    
    48
    -    if not workspace_dir:
    
    49
    -        workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
    
    50
    -    if not project_path:
    
    51
    -        project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    52
    -    else:
    
    53
    -        shutil.copytree(os.path.join(datafiles.dirname, datafiles.basename), project_path)
    
    54
    -    bin_files_path = os.path.join(project_path, 'files', 'bin-files')
    
    55
    -    element_path = os.path.join(project_path, 'elements')
    
    56
    -    element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
    
    48
    +class WorkspaceCreater():
    
    49
    +    def __init__(self, cli, tmpdir, datafiles, project_path=None):
    
    50
    +        self.cli = cli
    
    51
    +        self.tmpdir = tmpdir
    
    52
    +        self.datafiles = datafiles
    
    53
    +
    
    54
    +        if not project_path:
    
    55
    +            project_path = os.path.join(datafiles.dirname, datafiles.basename)
    
    56
    +        else:
    
    57
    +            shutil.copytree(os.path.join(datafiles.dirname, datafiles.basename), project_path)
    
    58
    +
    
    59
    +        self.project_path = project_path
    
    60
    +        self.bin_files_path = os.path.join(project_path, 'files', 'bin-files')
    
    61
    +
    
    62
    +        self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd')
    
    63
    +
    
    64
    +    def create_workspace_element(self, kind, track, suffix='', workspace_dir=None,
    
    65
    +                                 element_attrs=None):
    
    66
    +        element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
    
    67
    +        element_path = os.path.join(self.project_path, 'elements')
    
    68
    +        if not workspace_dir:
    
    69
    +            workspace_dir = os.path.join(self.workspace_cmd, element_name.rstrip('.bst'))
    
    70
    +
    
    71
    +        # Create our repo object of the given source type with
    
    72
    +        # the bin files, and then collect the initial ref.
    
    73
    +        repo = create_repo(kind, str(self.tmpdir))
    
    74
    +        ref = repo.create(self.bin_files_path)
    
    75
    +        if track:
    
    76
    +            ref = None
    
    77
    +
    
    78
    +        # Write out our test target
    
    79
    +        element = {
    
    80
    +            'kind': 'import',
    
    81
    +            'sources': [
    
    82
    +                repo.source_config(ref=ref)
    
    83
    +            ]
    
    84
    +        }
    
    85
    +        if element_attrs:
    
    86
    +            element = {**element, **element_attrs}
    
    87
    +        _yaml.dump(element,
    
    88
    +                   os.path.join(element_path,
    
    89
    +                                element_name))
    
    90
    +        return element_name, element_path, workspace_dir
    
    57 91
     
    
    58
    -    # Create our repo object of the given source type with
    
    59
    -    # the bin files, and then collect the initial ref.
    
    60
    -    #
    
    61
    -    repo = create_repo(kind, str(tmpdir))
    
    62
    -    ref = repo.create(bin_files_path)
    
    63
    -    if track:
    
    64
    -        ref = None
    
    92
    +    def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None,
    
    93
    +                                  element_attrs=None):
    
    65 94
     
    
    66
    -    # Write out our test target
    
    67
    -    element = {
    
    68
    -        'kind': 'import',
    
    69
    -        'sources': [
    
    70
    -            repo.source_config(ref=ref)
    
    71
    -        ]
    
    72
    -    }
    
    73
    -    if element_attrs:
    
    74
    -        element = {**element, **element_attrs}
    
    75
    -    _yaml.dump(element,
    
    76
    -               os.path.join(element_path,
    
    77
    -                            element_name))
    
    95
    +        element_tuples = []
    
    78 96
     
    
    79
    -    # Assert that there is no reference, a track & fetch is needed
    
    80
    -    state = cli.get_element_state(project_path, element_name)
    
    81
    -    if track:
    
    82
    -        assert state == 'no reference'
    
    83
    -    else:
    
    84
    -        assert state == 'fetch needed'
    
    97
    +        if suffixs is None:
    
    98
    +            suffixs = ['', ] * len(kinds)
    
    99
    +        else:
    
    100
    +            if len(suffixs) != len(kinds):
    
    101
    +                raise "terable error"
    
    85 102
     
    
    86
    -    # Now open the workspace, this should have the effect of automatically
    
    87
    -    # tracking & fetching the source from the repo.
    
    88
    -    args = ['workspace', 'open']
    
    89
    -    if track:
    
    90
    -        args.append('--track')
    
    91
    -    args.extend([element_name, workspace_dir])
    
    92
    -    result = cli.run(project=project_path, args=args)
    
    103
    +        for suffix, kind in zip(suffixs, kinds):
    
    104
    +            element_name, element_path, workspace_dir = \
    
    105
    +                self.create_workspace_element(kind, track, suffix, workspace_dir_usr,
    
    106
    +                                              element_attrs)
    
    93 107
     
    
    94
    -    result.assert_success()
    
    108
    +            # Assert that there is no reference, a track & fetch is needed
    
    109
    +            state = self.cli.get_element_state(self.project_path, element_name)
    
    110
    +            if track:
    
    111
    +                assert state == 'no reference'
    
    112
    +            else:
    
    113
    +                assert state == 'fetch needed'
    
    114
    +            element_tuples.append((element_name, workspace_dir))
    
    95 115
     
    
    96
    -    # Assert that we are now buildable because the source is
    
    97
    -    # now cached.
    
    98
    -    assert cli.get_element_state(project_path, element_name) == 'buildable'
    
    116
    +        return element_tuples
    
    99 117
     
    
    100
    -    # Check that the executable hello file is found in the workspace
    
    101
    -    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    
    102
    -    assert os.path.exists(filename)
    
    118
    +    def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None,
    
    119
    +                        element_attrs=None):
    
    120
    +
    
    121
    +        element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir,
    
    122
    +                                                        element_attrs)
    
    123
    +        os.makedirs(self.workspace_cmd, exist_ok=True)
    
    124
    +
    
    125
    +        # Now open the workspace, this should have the effect of automatically
    
    126
    +        # tracking & fetching the source from the repo.
    
    127
    +        args = ['workspace', 'open']
    
    128
    +        if track:
    
    129
    +            args.append('--track')
    
    130
    +        if workspace_dir is not None:
    
    131
    +            assert len(element_tuples) == 1, "test logic error"
    
    132
    +            _, workspace_dir = element_tuples[0]
    
    133
    +            args.extend(['--directory', workspace_dir])
    
    134
    +
    
    135
    +        args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    
    136
    +        result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args)
    
    103 137
     
    
    104
    -    return (element_name, project_path, workspace_dir)
    
    138
    +        result.assert_success()
    
    139
    +
    
    140
    +        for element_name, workspace_dir in element_tuples:
    
    141
    +            # Assert that we are now buildable because the source is
    
    142
    +            # now cached.
    
    143
    +            assert self.cli.get_element_state(self.project_path, element_name) == 'buildable'
    
    144
    +
    
    145
    +            # Check that the executable hello file is found in the workspace
    
    146
    +            filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    
    147
    +            assert os.path.exists(filename)
    
    148
    +
    
    149
    +        return element_tuples
    
    150
    +
    
    151
    +
    
    152
    +def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None,
    
    153
    +                   project_path=None, element_attrs=None):
    
    154
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles, project_path)
    
    155
    +    workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir,
    
    156
    +                                                  element_attrs)
    
    157
    +    assert len(workspaces) == 1
    
    158
    +    element_name, workspace = workspaces[0]
    
    159
    +    return element_name, workspace_object.project_path, workspace
    
    105 160
     
    
    106 161
     
    
    107 162
     @pytest.mark.datafiles(DATA_DIR)
    
    ... ... @@ -128,6 +183,103 @@ def test_open_bzr_customize(cli, tmpdir, datafiles):
    128 183
         assert(expected_output_str in str(output))
    
    129 184
     
    
    130 185
     
    
    186
    +@pytest.mark.datafiles(DATA_DIR)
    
    187
    +def test_open_multi(cli, tmpdir, datafiles):
    
    188
    +
    
    189
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    
    190
    +    workspaces = workspace_object.open_workspaces(repo_kinds, False)
    
    191
    +
    
    192
    +    for (elname, workspace), kind in zip(workspaces, repo_kinds):
    
    193
    +        assert kind in elname
    
    194
    +        workspace_lsdir = os.listdir(workspace)
    
    195
    +        if kind == 'git':
    
    196
    +            assert('.git' in workspace_lsdir)
    
    197
    +        elif kind == 'bzr':
    
    198
    +            assert('.bzr' in workspace_lsdir)
    
    199
    +        else:
    
    200
    +            assert not ('.git' in workspace_lsdir)
    
    201
    +            assert not ('.bzr' in workspace_lsdir)
    
    202
    +
    
    203
    +
    
    204
    +@pytest.mark.datafiles(DATA_DIR)
    
    205
    +def test_open_multi_unwritable(cli, tmpdir, datafiles):
    
    206
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    
    207
    +
    
    208
    +    element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds)
    
    209
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    
    210
    +
    
    211
    +    # Now open the workspace, this should have the effect of automatically
    
    212
    +    # tracking & fetching the source from the repo.
    
    213
    +    args = ['workspace', 'open']
    
    214
    +    args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    
    215
    +    cli.configure({'workspacedir': workspace_object.workspace_cmd})
    
    216
    +
    
    217
    +    cwdstat = os.stat(workspace_object.workspace_cmd)
    
    218
    +    try:
    
    219
    +        os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode - stat.S_IWRITE)
    
    220
    +        result = workspace_object.cli.run(project=workspace_object.project_path, args=args)
    
    221
    +    finally:
    
    222
    +        # Using this finally to make sure we always put thing back how they should be.
    
    223
    +        os.chmod(workspace_object.workspace_cmd, cwdstat.st_mode)
    
    224
    +
    
    225
    +    result.assert_main_error(ErrorDomain.STREAM, None)
    
    226
    +    # Normally we avoid checking stderr in favour of using the mechine readable result.assert_main_error
    
    227
    +    # But Tristan was very keen that the names of the elements left needing workspaces were present in the out put
    
    228
    +    assert (" ".join([element_name for element_name, workspace_dir_suffix in element_tuples[1:]]) in result.stderr)
    
    229
    +
    
    230
    +
    
    231
    +@pytest.mark.datafiles(DATA_DIR)
    
    232
    +def test_open_multi_with_directory(cli, tmpdir, datafiles):
    
    233
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    
    234
    +
    
    235
    +    element_tuples = workspace_object.create_workspace_elements(repo_kinds, False, repo_kinds)
    
    236
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    
    237
    +
    
    238
    +    # Now open the workspace, this should have the effect of automatically
    
    239
    +    # tracking & fetching the source from the repo.
    
    240
    +    args = ['workspace', 'open']
    
    241
    +    args.extend(['--directory', 'any/dir/should/fail'])
    
    242
    +
    
    243
    +    args.extend([element_name for element_name, workspace_dir_suffix in element_tuples])
    
    244
    +    result = workspace_object.cli.run(cwd=workspace_object.workspace_cmd, project=workspace_object.project_path,
    
    245
    +                                      args=args)
    
    246
    +
    
    247
    +    result.assert_main_error(ErrorDomain.ARTIFACT, None)
    
    248
    +    assert ("Directory option can only be used if a single element is given" in result.stderr)
    
    249
    +
    
    250
    +
    
    251
    +@pytest.mark.datafiles(DATA_DIR)
    
    252
    +def test_open_defaultlocation(cli, tmpdir, datafiles):
    
    253
    +    workspace_object = WorkspaceCreater(cli, tmpdir, datafiles)
    
    254
    +
    
    255
    +    ((element_name, workspace_dir), ) = workspace_object.create_workspace_elements(['git'], False, ['git'])
    
    256
    +    os.makedirs(workspace_object.workspace_cmd, exist_ok=True)
    
    257
    +
    
    258
    +    # Now open the workspace, this should have the effect of automatically
    
    259
    +    # tracking & fetching the source from the repo.
    
    260
    +    args = ['workspace', 'open']
    
    261
    +    args.append(element_name)
    
    262
    +
    
    263
    +    # In the other tests we set the cmd to workspace_object.workspace_cmd with the optional
    
    264
    +    # argument, cwd for the workspace_object.cli.run function. But hear we set the default
    
    265
    +    # workspace location to workspace_object.workspace_cmd and run the cli.run function with
    
    266
    +    # no cwd option so that it runs in the project directory.
    
    267
    +    cli.configure({'workspacedir': workspace_object.workspace_cmd})
    
    268
    +    result = workspace_object.cli.run(project=workspace_object.project_path,
    
    269
    +                                      args=args)
    
    270
    +
    
    271
    +    result.assert_success()
    
    272
    +
    
    273
    +    assert cli.get_element_state(workspace_object.project_path, element_name) == 'buildable'
    
    274
    +
    
    275
    +    # Check that the executable hello file is found in the workspace
    
    276
    +    # even though the cli.run function was not run with cwd = workspace_object.workspace_cmd
    
    277
    +    # the workspace should be created in there as we used the 'workspacedir' configuration
    
    278
    +    # option.
    
    279
    +    filename = os.path.join(workspace_dir, 'usr', 'bin', 'hello')
    
    280
    +    assert os.path.exists(filename)
    
    281
    +
    
    282
    +
    
    131 283
     @pytest.mark.datafiles(DATA_DIR)
    
    132 284
     @pytest.mark.parametrize("kind", repo_kinds)
    
    133 285
     def test_open_track(cli, tmpdir, datafiles, kind):
    
    ... ... @@ -150,7 +302,7 @@ def test_open_force(cli, tmpdir, datafiles, kind):
    150 302
     
    
    151 303
         # Now open the workspace again with --force, this should happily succeed
    
    152 304
         result = cli.run(project=project, args=[
    
    153
    -        'workspace', 'open', '--force', element_name, workspace
    
    305
    +        'workspace', 'open', '--force', '--directory', workspace, element_name
    
    154 306
         ])
    
    155 307
         result.assert_success()
    
    156 308
     
    
    ... ... @@ -165,7 +317,7 @@ def test_open_force_open(cli, tmpdir, datafiles, kind):
    165 317
     
    
    166 318
         # Now open the workspace again with --force, this should happily succeed
    
    167 319
         result = cli.run(project=project, args=[
    
    168
    -        'workspace', 'open', '--force', element_name, workspace
    
    320
    +        'workspace', 'open', '--force', '--directory', workspace, element_name
    
    169 321
         ])
    
    170 322
         result.assert_success()
    
    171 323
     
    
    ... ... @@ -196,7 +348,7 @@ def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
    196 348
     
    
    197 349
         # Now open the workspace again with --force, this should happily succeed
    
    198 350
         result = cli.run(project=project, args=[
    
    199
    -        'workspace', 'open', '--force', element_name2, workspace
    
    351
    +        'workspace', 'open', '--force', '--directory', workspace, element_name2
    
    200 352
         ])
    
    201 353
     
    
    202 354
         # Assert that the file in workspace 1 has been replaced
    
    ... ... @@ -504,7 +656,7 @@ def test_buildable_no_ref(cli, tmpdir, datafiles):
    504 656
         # Now open the workspace. We don't need to checkout the source though.
    
    505 657
         workspace = os.path.join(str(tmpdir), 'workspace-no-ref')
    
    506 658
         os.makedirs(workspace)
    
    507
    -    args = ['workspace', 'open', '--no-checkout', element_name, workspace]
    
    659
    +    args = ['workspace', 'open', '--no-checkout', '--directory', workspace, element_name]
    
    508 660
         result = cli.run(project=project, args=args)
    
    509 661
         result.assert_success()
    
    510 662
     
    
    ... ... @@ -766,7 +918,7 @@ def test_list_supported_workspace(cli, tmpdir, datafiles, workspace_cfg, expecte
    766 918
                                 element_name))
    
    767 919
     
    
    768 920
         # Make a change to the workspaces file
    
    769
    -    result = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    921
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    770 922
         result.assert_success()
    
    771 923
         result = cli.run(project=project, args=['workspace', 'close', '--remove-dir', element_name])
    
    772 924
         result.assert_success()
    

  • tests/integration/pip_source.py
    ... ... @@ -4,6 +4,7 @@ import pytest
    4 4
     from buildstream import _yaml
    
    5 5
     
    
    6 6
     from tests.testutils import cli_integration as cli
    
    7
    +from tests.testutils.python_repo import setup_pypi_repo
    
    7 8
     from tests.testutils.integration import assert_contains
    
    8 9
     
    
    9 10
     
    
    ... ... @@ -17,12 +18,21 @@ DATA_DIR = os.path.join(
    17 18
     
    
    18 19
     
    
    19 20
     @pytest.mark.datafiles(DATA_DIR)
    
    20
    -def test_pip_source_import(cli, tmpdir, datafiles):
    
    21
    +def test_pip_source_import(cli, tmpdir, datafiles, setup_pypi_repo):
    
    21 22
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    22 23
         checkout = os.path.join(cli.directory, 'checkout')
    
    23 24
         element_path = os.path.join(project, 'elements')
    
    24 25
         element_name = 'pip/hello.bst'
    
    25 26
     
    
    27
    +    # check that exotically named packages are imported correctly
    
    28
    +    myreqs_packages = ['hellolib']
    
    29
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    
    30
    +
    
    31
    +    # create mock pypi repository
    
    32
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    
    33
    +    os.makedirs(pypi_repo, exist_ok=True)
    
    34
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    
    35
    +
    
    26 36
         element = {
    
    27 37
             'kind': 'import',
    
    28 38
             'sources': [
    
    ... ... @@ -32,9 +42,9 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    32 42
                 },
    
    33 43
                 {
    
    34 44
                     'kind': 'pip',
    
    35
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    45
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    
    36 46
                     'requirements-files': ['myreqs.txt'],
    
    37
    -                'packages': ['app2']
    
    47
    +                'packages': packages
    
    38 48
                 }
    
    39 49
             ]
    
    40 50
         }
    
    ... ... @@ -51,16 +61,31 @@ def test_pip_source_import(cli, tmpdir, datafiles):
    51 61
         assert result.exit_code == 0
    
    52 62
     
    
    53 63
         assert_contains(checkout, ['/.bst_pip_downloads',
    
    54
    -                               '/.bst_pip_downloads/HelloLib-0.1.tar.gz',
    
    55
    -                               '/.bst_pip_downloads/App2-0.1.tar.gz'])
    
    64
    +                               '/.bst_pip_downloads/hellolib-0.1.tar.gz',
    
    65
    +                               '/.bst_pip_downloads/app2-0.1.tar.gz',
    
    66
    +                               '/.bst_pip_downloads/app.3-0.1.tar.gz',
    
    67
    +                               '/.bst_pip_downloads/app-4-0.1.tar.gz',
    
    68
    +                               '/.bst_pip_downloads/app_5-0.1.tar.gz',
    
    69
    +                               '/.bst_pip_downloads/app.no.6-0.1.tar.gz',
    
    70
    +                               '/.bst_pip_downloads/app-no-7-0.1.tar.gz',
    
    71
    +                               '/.bst_pip_downloads/app_no_8-0.1.tar.gz'])
    
    56 72
     
    
    57 73
     
    
    58 74
     @pytest.mark.datafiles(DATA_DIR)
    
    59
    -def test_pip_source_build(cli, tmpdir, datafiles):
    
    75
    +def test_pip_source_build(cli, tmpdir, datafiles, setup_pypi_repo):
    
    60 76
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    61 77
         element_path = os.path.join(project, 'elements')
    
    62 78
         element_name = 'pip/hello.bst'
    
    63 79
     
    
    80
    +    # check that exotically named packages are imported correctly
    
    81
    +    myreqs_packages = ['hellolib']
    
    82
    +    packages = ['app2', 'app.3', 'app-4', 'app_5', 'app.no.6', 'app-no-7', 'app_no_8']
    
    83
    +
    
    84
    +    # create mock pypi repository
    
    85
    +    pypi_repo = os.path.join(project, 'files', 'pypi-repo')
    
    86
    +    os.makedirs(pypi_repo, exist_ok=True)
    
    87
    +    setup_pypi_repo(myreqs_packages + packages, pypi_repo)
    
    88
    +
    
    64 89
         element = {
    
    65 90
             'kind': 'manual',
    
    66 91
             'depends': ['base.bst'],
    
    ... ... @@ -71,16 +96,15 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    71 96
                 },
    
    72 97
                 {
    
    73 98
                     'kind': 'pip',
    
    74
    -                'url': 'file://{}'.format(os.path.realpath(os.path.join(project, 'files', 'pypi-repo'))),
    
    99
    +                'url': 'file://{}'.format(os.path.realpath(pypi_repo)),
    
    75 100
                     'requirements-files': ['myreqs.txt'],
    
    76
    -                'packages': ['app2']
    
    101
    +                'packages': packages
    
    77 102
                 }
    
    78 103
             ],
    
    79 104
             'config': {
    
    80 105
                 'install-commands': [
    
    81 106
                     'pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz',
    
    82
    -                'chmod +x app1.py',
    
    83
    -                'install app1.py  %{install-root}/usr/bin/'
    
    107
    +                'install app1.py %{install-root}/usr/bin/'
    
    84 108
                 ]
    
    85 109
             }
    
    86 110
         }
    
    ... ... @@ -95,5 +119,4 @@ def test_pip_source_build(cli, tmpdir, datafiles):
    95 119
     
    
    96 120
         result = cli.run(project=project, args=['shell', element_name, '/usr/bin/app1.py'])
    
    97 121
         assert result.exit_code == 0
    
    98
    -    assert result.output == """Hello App1!
    
    99
    -"""
    122
    +    assert result.output == "Hello App1! This is hellolib\n"

  • tests/integration/project/files/pypi-repo/app2/App2-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/app2/index.html deleted
    1
    -<html>
    
    2
    -  <head>
    
    3
    -    <title>Links for app1</title>
    
    4
    -  </head>
    
    5
    -  <body>
    
    6
    -    <a href="">'App2-0.1.tar.gz'>App2-0.1.tar.gz</a><br />
    
    7
    -  </body>
    
    8
    -</html>

  • tests/integration/project/files/pypi-repo/hellolib/HelloLib-0.1.tar.gz deleted
    No preview for this file type
  • tests/integration/project/files/pypi-repo/hellolib/index.html deleted
    1
    -<html>
    
    2
    -  <head>
    
    3
    -    <title>Links for app1</title>
    
    4
    -  </head>
    
    5
    -  <body>
    
    6
    -    <a href="">'HelloLib-0.1.tar.gz'>HelloLib-0.1.tar.gz</a><br />
    
    7
    -  </body>
    
    8
    -</html>

  • tests/integration/shell.py
    ... ... @@ -278,7 +278,7 @@ def test_workspace_visible(cli, tmpdir, datafiles):
    278 278
     
    
    279 279
         # Open a workspace on our build failing element
    
    280 280
         #
    
    281
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    281
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    282 282
         assert res.exit_code == 0
    
    283 283
     
    
    284 284
         # Ensure the dependencies of our build failing element are built
    

  • tests/integration/workspace.py
    ... ... @@ -23,7 +23,7 @@ def test_workspace_mount(cli, tmpdir, datafiles):
    23 23
         workspace = os.path.join(cli.directory, 'workspace')
    
    24 24
         element_name = 'workspace/workspace-mount.bst'
    
    25 25
     
    
    26
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    26
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    27 27
         assert res.exit_code == 0
    
    28 28
     
    
    29 29
         res = cli.run(project=project, args=['build', element_name])
    
    ... ... @@ -39,7 +39,7 @@ def test_workspace_commanddir(cli, tmpdir, datafiles):
    39 39
         workspace = os.path.join(cli.directory, 'workspace')
    
    40 40
         element_name = 'workspace/workspace-commanddir.bst'
    
    41 41
     
    
    42
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    42
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    43 43
         assert res.exit_code == 0
    
    44 44
     
    
    45 45
         res = cli.run(project=project, args=['build', element_name])
    
    ... ... @@ -75,7 +75,7 @@ def test_workspace_updated_dependency(cli, tmpdir, datafiles):
    75 75
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    
    76 76
     
    
    77 77
         # First open the workspace
    
    78
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    78
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    79 79
         assert res.exit_code == 0
    
    80 80
     
    
    81 81
         # We build the workspaced element, so that we have an artifact
    
    ... ... @@ -130,7 +130,7 @@ def test_workspace_update_dependency_failed(cli, tmpdir, datafiles):
    130 130
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    
    131 131
     
    
    132 132
         # First open the workspace
    
    133
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    133
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    134 134
         assert res.exit_code == 0
    
    135 135
     
    
    136 136
         # We build the workspaced element, so that we have an artifact
    
    ... ... @@ -205,7 +205,7 @@ def test_updated_dependency_nested(cli, tmpdir, datafiles):
    205 205
         _yaml.dump(dependency, os.path.join(element_path, dep_name))
    
    206 206
     
    
    207 207
         # First open the workspace
    
    208
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    208
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    209 209
         assert res.exit_code == 0
    
    210 210
     
    
    211 211
         # We build the workspaced element, so that we have an artifact
    
    ... ... @@ -258,7 +258,7 @@ def test_incremental_configure_commands_run_only_once(cli, tmpdir, datafiles):
    258 258
         _yaml.dump(element, os.path.join(element_path, element_name))
    
    259 259
     
    
    260 260
         # We open a workspace on the above element
    
    261
    -    res = cli.run(project=project, args=['workspace', 'open', element_name, workspace])
    
    261
    +    res = cli.run(project=project, args=['workspace', 'open', '--directory', workspace, element_name])
    
    262 262
         res.assert_success()
    
    263 263
     
    
    264 264
         # Then we build, and check whether the configure step succeeded
    

  • tests/plugins/filter.py
    ... ... @@ -108,7 +108,7 @@ def test_filter_forbid_also_rdep(datafiles, cli):
    108 108
     def test_filter_workspace_open(datafiles, cli, tmpdir):
    
    109 109
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    110 110
         workspace_dir = os.path.join(tmpdir.dirname, tmpdir.basename, "workspace")
    
    111
    -    result = cli.run(project=project, args=['workspace', 'open', 'deps-permitted.bst', workspace_dir])
    
    111
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'deps-permitted.bst'])
    
    112 112
         result.assert_success()
    
    113 113
         assert os.path.exists(os.path.join(workspace_dir, "foo"))
    
    114 114
         assert os.path.exists(os.path.join(workspace_dir, "bar"))
    
    ... ... @@ -120,7 +120,7 @@ def test_filter_workspace_build(datafiles, cli, tmpdir):
    120 120
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    121 121
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    
    122 122
         workspace_dir = os.path.join(tempdir, "workspace")
    
    123
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    
    123
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    
    124 124
         result.assert_success()
    
    125 125
         src = os.path.join(workspace_dir, "foo")
    
    126 126
         dst = os.path.join(workspace_dir, "quux")
    
    ... ... @@ -138,7 +138,7 @@ def test_filter_workspace_close(datafiles, cli, tmpdir):
    138 138
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    139 139
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    
    140 140
         workspace_dir = os.path.join(tempdir, "workspace")
    
    141
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    
    141
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    
    142 142
         result.assert_success()
    
    143 143
         src = os.path.join(workspace_dir, "foo")
    
    144 144
         dst = os.path.join(workspace_dir, "quux")
    
    ... ... @@ -158,7 +158,7 @@ def test_filter_workspace_reset(datafiles, cli, tmpdir):
    158 158
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    159 159
         tempdir = os.path.join(tmpdir.dirname, tmpdir.basename)
    
    160 160
         workspace_dir = os.path.join(tempdir, "workspace")
    
    161
    -    result = cli.run(project=project, args=['workspace', 'open', 'output-orphans.bst', workspace_dir])
    
    161
    +    result = cli.run(project=project, args=['workspace', 'open', '--directory', workspace_dir, 'output-orphans.bst'])
    
    162 162
         result.assert_success()
    
    163 163
         src = os.path.join(workspace_dir, "foo")
    
    164 164
         dst = os.path.join(workspace_dir, "quux")
    

  • tests/sources/pip.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
     
    
    4 4
     from buildstream._exceptions import ErrorDomain
    
    5 5
     from buildstream import _yaml
    
    6
    +from buildstream.plugins.sources.pip import _match_package_name
    
    6 7
     from tests.testutils import cli
    
    7 8
     
    
    8 9
     DATA_DIR = os.path.join(
    
    ... ... @@ -45,3 +46,22 @@ def test_no_packages(cli, tmpdir, datafiles):
    45 46
             'show', 'target.bst'
    
    46 47
         ])
    
    47 48
         result.assert_main_error(ErrorDomain.SOURCE, None)
    
    49
    +
    
    50
    +
    
    51
    +# Test that pip source parses tar ball names correctly for the ref
    
    52
    +@pytest.mark.parametrize(
    
    53
    +    'tarball, expected_name, expected_version',
    
    54
    +    [
    
    55
    +        ('dotted.package-0.9.8.tar.gz', 'dotted.package', '0.9.8'),
    
    56
    +        ('hyphenated-package-2.6.0.tar.gz', 'hyphenated-package', '2.6.0'),
    
    57
    +        ('underscore_pkg-3.1.0.tar.gz', 'underscore_pkg', '3.1.0'),
    
    58
    +        ('numbers2and5-1.0.1.tar.gz', 'numbers2and5', '1.0.1'),
    
    59
    +        ('multiple.dots.package-5.6.7.tar.gz', 'multiple.dots.package', '5.6.7'),
    
    60
    +        ('multiple-hyphens-package-1.2.3.tar.gz', 'multiple-hyphens-package', '1.2.3'),
    
    61
    +        ('multiple_underscore_pkg-3.4.5.tar.gz', 'multiple_underscore_pkg', '3.4.5'),
    
    62
    +        ('shortversion-1.0.tar.gz', 'shortversion', '1.0'),
    
    63
    +        ('longversion-1.2.3.4.tar.gz', 'longversion', '1.2.3.4')
    
    64
    +    ])
    
    65
    +def test_match_package_name(tarball, expected_name, expected_version):
    
    66
    +    name, version = _match_package_name(tarball)
    
    67
    +    assert (expected_name, expected_version) == (name, version)

  • tests/testutils/__init__.py
    ... ... @@ -29,3 +29,4 @@ from .artifactshare import create_artifact_share
    29 29
     from .element_generators import create_element_size, update_element_size
    
    30 30
     from .junction import generate_junction
    
    31 31
     from .runner_integration import wait_for_cache_granularity
    
    32
    +from .python_repo import setup_pypi_repo

  • tests/testutils/python_repo.py
    1
    +from setuptools.sandbox import run_setup
    
    2
    +import os
    
    3
    +import pytest
    
    4
    +import re
    
    5
    +import shutil
    
    6
    +
    
    7
    +
    
    8
    +SETUP_TEMPLATE = '''\
    
    9
    +from setuptools import setup
    
    10
    +
    
    11
    +setup(
    
    12
    +    name='{name}',
    
    13
    +    version='{version}',
    
    14
    +    description='{name}',
    
    15
    +    packages=['{pkgdirname}'],
    
    16
    +    entry_points={{
    
    17
    +        'console_scripts': [
    
    18
    +            '{pkgdirname}={pkgdirname}:main'
    
    19
    +        ]
    
    20
    +    }}
    
    21
    +)
    
    22
    +'''
    
    23
    +
    
    24
    +# All packages generated via generate_pip_package will have the functions below
    
    25
    +INIT_TEMPLATE = '''\
    
    26
    +def main():
    
    27
    +    print('This is {name}')
    
    28
    +
    
    29
    +def hello(actor='world'):
    
    30
    +    print('Hello {{}}! This is {name}'.format(actor))
    
    31
    +'''
    
    32
    +
    
    33
    +HTML_TEMPLATE = '''\
    
    34
    +<html>
    
    35
    +  <head>
    
    36
    +    <title>Links for {name}</title>
    
    37
    +  </head>
    
    38
    +  <body>
    
    39
    +    <a href=''>{name}-{version}.tar.gz</a><br />
    
    40
    +  </body>
    
    41
    +</html>
    
    42
    +'''
    
    43
    +
    
    44
    +
    
    45
    +# Creates a simple python source distribution and copies this into a specified
    
    46
    +# directory which is to serve as a mock python repository
    
    47
    +#
    
    48
    +# Args:
    
    49
    +#    tmpdir (str): Directory in which the source files will be created
    
    50
    +#    pypi (str): Directory serving as a mock python repository
    
    51
    +#    name (str): The name of the package to be created
    
    52
    +#    version (str): The version of the package to be created
    
    53
    +#
    
    54
    +# Returns:
    
    55
    +#    None
    
    56
    +#
    
    57
    +def generate_pip_package(tmpdir, pypi, name, version='0.1'):
    
    58
    +    # check if package already exists in pypi
    
    59
    +    pypi_package = os.path.join(pypi, re.sub('[^0-9a-zA-Z]+', '-', name))
    
    60
    +    if os.path.exists(pypi_package):
    
    61
    +        return
    
    62
    +
    
    63
    +    # create the package source files in tmpdir resulting in a directory
    
    64
    +    # tree resembling the following structure:
    
    65
    +    #
    
    66
    +    # tmpdir
    
    67
    +    # |-- setup.py
    
    68
    +    # `-- package
    
    69
    +    #     `-- __init__.py
    
    70
    +    #
    
    71
    +    setup_file = os.path.join(tmpdir, 'setup.py')
    
    72
    +    pkgdirname = re.sub('[^0-9a-zA-Z]+', '', name)
    
    73
    +    with open(setup_file, 'w') as f:
    
    74
    +        f.write(
    
    75
    +            SETUP_TEMPLATE.format(
    
    76
    +                name=name,
    
    77
    +                version=version,
    
    78
    +                pkgdirname=pkgdirname
    
    79
    +            )
    
    80
    +        )
    
    81
    +    os.chmod(setup_file, 0o755)
    
    82
    +
    
    83
    +    package = os.path.join(tmpdir, pkgdirname)
    
    84
    +    os.makedirs(package)
    
    85
    +
    
    86
    +    main_file = os.path.join(package, '__init__.py')
    
    87
    +    with open(main_file, 'w') as f:
    
    88
    +        f.write(INIT_TEMPLATE.format(name=name))
    
    89
    +    os.chmod(main_file, 0o644)
    
    90
    +
    
    91
    +    run_setup(setup_file, ['sdist'])
    
    92
    +
    
    93
    +    # create directory for this package in pypi resulting in a directory
    
    94
    +    # tree resembling the following structure:
    
    95
    +    #
    
    96
    +    # pypi
    
    97
    +    # `-- pypi_package
    
    98
    +    #     |-- index.html
    
    99
    +    #     `-- foo-0.1.tar.gz
    
    100
    +    #
    
    101
    +    os.makedirs(pypi_package)
    
    102
    +
    
    103
    +    # add an index html page
    
    104
    +    index_html = os.path.join(pypi_package, 'index.html')
    
    105
    +    with open(index_html, 'w') as f:
    
    106
    +        f.write(HTML_TEMPLATE.format(name=name, version=version))
    
    107
    +
    
    108
    +    # copy generated tarfile to pypi package
    
    109
    +    dist_dir = os.path.join(tmpdir, 'dist')
    
    110
    +    for tar in os.listdir(dist_dir):
    
    111
    +        tarpath = os.path.join(dist_dir, tar)
    
    112
    +        shutil.copy(tarpath, pypi_package)
    
    113
    +
    
    114
    +
    
    115
    +@pytest.fixture
    
    116
    +def setup_pypi_repo(tmpdir):
    
    117
    +    def create_pkgdir(package):
    
    118
    +        pkgdirname = re.sub('[^0-9a-zA-Z]+', '', package)
    
    119
    +        pkgdir = os.path.join(str(tmpdir), pkgdirname)
    
    120
    +        os.makedirs(pkgdir)
    
    121
    +        return pkgdir
    
    122
    +
    
    123
    +    def add_packages(packages, pypi_repo):
    
    124
    +        for package in packages:
    
    125
    +            pkgdir = create_pkgdir(package)
    
    126
    +            generate_pip_package(pkgdir, pypi_repo, package)
    
    127
    +
    
    128
    +    return add_packages



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]