GNOME.org
 

[Notes] [Git][BuildStream/buildstream][edbaunton/executable-remote-source] 18 commits: element.py (docs): dashes not underscores for build and install root

Title: GitLab

Tristan Van Berkom pushed to branch edbaunton/executable-remote-source at BuildStream / buildstream

Commits:

16 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -143,7 +143,6 @@ docs:
    143 143 
       - pip3 install sphinx-click
    144 144 
       - pip3 install sphinx_rtd_theme
    145 145 
       - cd dist && ./unpack.sh && cd buildstream
    146 
    -  - pip3 install .
    147 146 
       - make BST_FORCE_SESSION_REBUILD=1 -C doc
    148 147 
       - cd ../..
    149 148 
       - mv dist/buildstream/doc/build/html public

  • .gitlab/issue_templates/bst_task.md
    ... ... @@ -6,9 +6,9 @@
    6 6
    7 7 
     [//]: # (Short summary of the action to be executed)
    8 8
    9 
    -* [  ] Action 1
    10 
    -* [  ] Action 2
    11 
    -* [  ] Action 3
    9 
    +* [ ] Action 1
    10 
    +* [ ] Action 2
    11 
    +* [ ] Action 3
    12 12
    13 13 
     ## Acceptance Criteria
    14 14

  • HACKING.rst
    ... ... @@ -261,9 +261,6 @@ using pip or some other mechanism::
    261 261 
       # Additional optional dependencies required
    262 262 
       pip3 install --user arpy
    263 263
    264 
    -Furthermore, the documentation build requires that BuildStream itself
    265 
    -be installed, as it will be used in the process of generating its docs.
    266 
    -
    267 264 
     To build the documentation, just run the following::
    268 265
    269 266 
       make -C doc

  • buildstream/__main__.py
    1 
    +##################################################################
    2 
    +#                      Private Entry Point                       #
    3 
    +##################################################################
    4 
    +#
    5 
    +# This allows running the cli when BuildStream is uninstalled,
    6 
    +# as long as BuildStream repo is in PYTHONPATH, one can run it
    7 
    +# with:
    8 
    +#
    9 
    +#    python3 -m buildstream [program args]
    10 
    +#
    11 
    +# This is used when we need to run BuildStream before installing,
    12 
    +# like when we build documentation.
    13 
    +#
    14 
    +if __name__ == '__main__':
    15 
    +    # pylint: disable=no-value-for-parameter
    16 
    +    from ._frontend.cli import cli
    17 
    +    cli()

  • buildstream/_scheduler/queues/pullqueue.py
    ... ... @@ -29,7 +29,7 @@ class PullQueue(Queue):
    29 29
    30 30 
         action_name = "Pull"
    31 31 
         complete_name = "Pulled"
    32 
    -    resources = [ResourceType.UPLOAD]
    32 
    +    resources = [ResourceType.DOWNLOAD]
    33 33
    34 34 
         def process(self, element):
    35 35 
             # returns whether an artifact was downloaded or not

  • buildstream/_versions.py
    ... ... @@ -23,7 +23,7 @@
    23 23 
     # This version is bumped whenever enhancements are made
    24 24 
     # to the `project.conf` format or the core element format.
    25 25 
     #
    26 
    -BST_FORMAT_VERSION = 12
    26 
    +BST_FORMAT_VERSION = 13
    27 27
    28 28
    29 29 
     # The base BuildStream artifact version

  • buildstream/data/userconfig.yaml
    ... ... @@ -35,13 +35,13 @@ cache:
    35 35 
     #
    36 36 
     scheduler:
    37 37
    38 
    -  # Maximum number of simultaneous source downloading tasks.
    38 
    +  # Maximum number of simultaneous downloading tasks.
    39 39 
       fetchers: 10
    40 40
    41 41 
       # Maximum number of simultaneous build tasks.
    42 42 
       builders: 4
    43 43
    44 
    -  # Maximum number of simultaneous artifact uploading tasks.
    44 
    +  # Maximum number of simultaneous uploading tasks.
    45 45 
       pushers: 4
    46 46
    47 47 
       # Maximum number of retries for network tasks.

  • buildstream/element.py
    ... ... @@ -369,8 +369,8 @@ class Element(Plugin):
    369 369 
             generated script is run:
    370 370
    371 371 
             - All element variables have been exported.
    372 
    -        - The cwd is `self.get_variable('build_root')/self.normal_name`.
    373 
    -        - $PREFIX is set to `self.get_variable('install_root')`.
    372 
    +        - The cwd is `self.get_variable('build-root')/self.normal_name`.
    373 
    +        - $PREFIX is set to `self.get_variable('install-root')`.
    374 374 
             - The directory indicated by $PREFIX is an empty directory.
    375 375
    376 376 
             Files are expected to be installed to $PREFIX.

  • buildstream/plugins/sources/remote.py
    ... ... @@ -35,6 +35,10 @@ remote - stage files from remote urls
    35 35 
        # If not specified, the basename of the url will be used.
    36 36 
        # filename: customfilename
    37 37
    38 
    +   # Optionally specify whether the downloaded file should be
    39 
    +   # marked executable.
    40 
    +   # executable: true
    41 
    +
    38 42 
        # Specify the url. Using an alias defined in your project
    39 43 
        # configuration is encouraged. 'bst track' will update the
    40 44 
        # sha256sum in 'ref' to the downloaded file's sha256sum.
    ... ... @@ -43,6 +47,8 @@ remote - stage files from remote urls
    43 47 
        # Specify the ref. It's a sha256sum of the file you download.
    44 48 
        ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
    45 49
    50 
    +
    51 
    +
    46 52 
     .. note::
    47 53
    48 54 
        The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
    ... ... @@ -60,22 +66,31 @@ class RemoteSource(DownloadableFileSource):
    60 66 
             super().configure(node)
    61 67
    62 68 
             self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
    69 
    +        self.executable = self.node_get_member(node, bool, 'executable', False)
    63 70
    64 71 
             if os.sep in self.filename:
    65 72 
                 raise SourceError('{}: filename parameter cannot contain directories'.format(self),
    66 73 
                                   reason="filename-contains-directory")
    67 
    -        self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename'])
    74 
    +        self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
    68 75
    69 76 
         def get_unique_key(self):
    70 
    -        return super().get_unique_key() + [self.filename]
    77 
    +        return super().get_unique_key() + [self.filename, self.executable]
    71 78
    72 79 
         def stage(self, directory):
    73 80 
             # Same as in local plugin, don't use hardlinks to stage sources, they
    74 81 
             # are not write protected in the sandbox.
    75 82 
             dest = os.path.join(directory, self.filename)
    76 83 
             with self.timed_activity("Staging remote file to {}".format(dest)):
    84 
    +
    77 85 
                 utils.safe_copy(self._get_mirror_file(), dest)
    78 86
    87 
    +            # To prevent user's umask introducing variability here, explicitly set
    88 
    +            # file modes.
    89 
    +            if self.executable:
    90 
    +                os.chmod(dest, 0o755)
    91 
    +            else:
    92 
    +                os.chmod(dest, 0o644)
    93 
    +
    79 94
    80 95 
     def setup():
    81 96 
         return RemoteSource

  • doc/Makefile
    ... ... @@ -31,6 +31,9 @@ ifneq ($(strip $(BST_FORCE_SESSION_REBUILD)),)
    31 31 
     BST2HTMLOPTS = --force
    32 32 
     endif
    33 33
    34 
    +# Help Python find buildstream and its plugins
    35 
    +PYTHONPATH=$(CURDIR)/..:$(CURDIR)/../buildstream/plugins
    36 
    +
    34 37
    35 38 
     .PHONY: all clean templates templates-clean sessions sessions-prep sessions-clean html devhelp
    36 39
    ... ... @@ -65,7 +68,6 @@ define plugin-doc-skeleton
    65 68 
     endef
    66 69
    67 70
    68 
    -# We set PYTHONPATH here because source/conf.py sys.modules hacks dont seem to help sphinx-build import the plugins
    69 71 
     all: html devhelp
    70 72
    71 73 
     clean: templates-clean sessions-clean
    ... ... @@ -103,7 +105,7 @@ sessions-prep:
    103 105 
     #
    104 106 
     sessions: sessions-prep
    105 107 
     	for file in $(wildcard sessions/*.run); do		\
    106 
    -	    $(BST2HTML) $(BST2HTMLOPTS) --description $$file;	\
    108 
    +	    PYTHONPATH=$(PYTHONPATH) $(BST2HTML) $(BST2HTMLOPTS) $$file;	\
    107 109 
     	done
    108 110
    109 111 
     sessions-clean:
    ... ... @@ -114,7 +116,7 @@ sessions-clean:
    114 116 
     #
    115 117 
     html devhelp: templates sessions
    116 118 
     	@echo "Building $@..."
    117 
    -	PYTHONPATH=$(CURDIR)/../buildstream/plugins \
    119 
    +	PYTHONPATH=$(PYTHONPATH) \
    118 120 
     	    $(SPHINXBUILD) -b $@ $(ALLSPHINXOPTS) "$(BUILDDIR)/$@" \
    119 121 
     	    $(wildcard source/*.rst) \
    120 122 
     	    $(wildcard source/tutorial/*.rst) \

  • doc/bst2html.py
    ... ... @@ -204,7 +204,7 @@ def workdir(source_cache=None):
    204 204 
             yield (tempdir, bst_config_file, source_cache)
    205 205
    206 206
    207 
    -# run_command()
    207 
    +# run_bst_command()
    208 208 
     #
    209 209 
     # Runs a command
    210 210 
     #
    ... ... @@ -216,10 +216,30 @@ def workdir(source_cache=None):
    216 216 
     # Returns:
    217 217 
     #    (str): The colorized combined stdout/stderr of BuildStream
    218 218 
     #
    219 
    -def run_command(config_file, directory, command):
    220 
    -    click.echo("Running command in directory '{}': bst {}".format(directory, command), err=True)
    219 
    +def run_bst_command(config_file, directory, command):
    220 
    +    click.echo("Running bst command in directory '{}': bst {}".format(directory, command), err=True)
    221 221
    222 
    -    argv = ['bst', '--colors', '--config', config_file] + shlex.split(command)
    222 
    +    argv = ['python3', '-m', 'buildstream', '--colors', '--config', config_file] + shlex.split(command)
    223 
    +    p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    224 
    +    out, _ = p.communicate()
    225 
    +    return out.decode('utf-8').strip()
    226 
    +
    227 
    +
    228 
    +# run_shell_command()
    229 
    +#
    230 
    +# Runs a command
    231 
    +#
    232 
    +# Args:
    233 
    +#    directory (str): The project directory
    234 
    +#    command (str): A shell command
    235 
    +#
    236 
    +# Returns:
    237 
    +#    (str): The combined stdout/stderr of the shell command
    238 
    +#
    239 
    +def run_shell_command(directory, command):
    240 
    +    click.echo("Running shell command in directory '{}': {}".format(directory, command), err=True)
    241 
    +
    242 
    +    argv = shlex.split(command)
    223 243 
         p = subprocess.Popen(argv, cwd=directory, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
    224 244 
         out, _ = p.communicate()
    225 245 
         return out.decode('utf-8').strip()
    ... ... @@ -373,12 +393,18 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
    373 393 
             # Get the command string
    374 394 
             command_str = _yaml.node_get(command, str, 'command')
    375 395
    396 
    +        # Check whether this is a shell command and not a bst command
    397 
    +        is_shell = _yaml.node_get(command, bool, 'shell', default_value=False)
    398 
    +
    376 399 
             # Check if there is fake output
    377 400 
             command_fake_output = _yaml.node_get(command, str, 'fake-output', default_value=None)
    378 401
    379 402 
             # Run the command, or just use the fake output
    380 403 
             if command_fake_output is None:
    381 
    -            command_out = run_command(config_file, directory, command_str)
    404 
    +            if is_shell:
    405 
    +                command_out = run_shell_command(directory, command_str)
    406 
    +            else:
    407 
    +                command_out = run_bst_command(config_file, directory, command_str)
    382 408 
             else:
    383 409 
                 command_out = command_fake_output
    384 410
    ... ... @@ -414,14 +440,8 @@ def run_session(description, tempdir, source_cache, palette, config_file, force)
    414 440 
     @click.option('--palette', '-p', default='tango',
    415 441 
                   type=click.Choice(['solarized', 'solarized-xterm', 'tango', 'xterm', 'console']),
    416 442 
                   help="Selects a palette for the output style")
    417 
    -@click.option('--output', '-o',
    418 
    -              type=click.Path(file_okay=True, dir_okay=False, writable=True),
    419 
    -              help="A file to store the output")
    420 
    -@click.option('--description', '-d',
    421 
    -              type=click.Path(file_okay=True, dir_okay=False, readable=True),
    422 
    -              help="A file describing what to do")
    423 
    -@click.argument('command', type=click.STRING, nargs=-1)
    424 
    -def run_bst(directory, force, source_cache, description, palette, output, command):
    443 
    +@click.argument('description', click.Path(file_okay=True, dir_okay=False, readable=True))
    444 
    +def run_bst(directory, force, source_cache, description, palette):
    425 445 
         """Run a bst command and capture stdout/stderr in html
    426 446
    427 447 
         This command normally takes a description yaml file, see the HACKING
    ... ... @@ -430,45 +450,8 @@ def run_bst(directory, force, source_cache, description, palette, output, comman
    430 450 
         if not source_cache and os.environ.get('BST_SOURCE_CACHE'):
    431 451 
             source_cache = os.environ['BST_SOURCE_CACHE']
    432 452
    433 
    -    if output is not None and not check_needs_build(None, output, force=force):
    434 
    -        click.echo("No need to rebuild {}".format(output))
    435 
    -        return 0
    436 
    -
    437 453 
         with workdir(source_cache=source_cache) as (tempdir, config_file, source_cache):
    438 
    -
    439 
    -        if description:
    440 
    -            run_session(description, tempdir, source_cache, palette, config_file, force)
    441 
    -            return 0
    442 
    -
    443 
    -        # Run a command specified on the CLI
    444 
    -        #
    445 
    -        if not directory:
    446 
    -            directory = os.getcwd()
    447 
    -        else:
    448 
    -            directory = os.path.abspath(directory)
    449 
    -            directory = os.path.realpath(directory)
    450 
    -
    451 
    -        if not command:
    452 
    -            command = []
    453 
    -        command_str = " ".join(command)
    454 
    -
    455 
    -        # Run the command
    456 
    -        #
    457 
    -        command_out = run_command(config_file, directory, command_str)
    458 
    -
    459 
    -        # Generate a nice html div for this output
    460 
    -        #
    461 
    -        converted = generate_html(command_out, directory, config_file,
    462 
    -                                  source_cache, tempdir, palette,
    463 
    -                                  command_str)
    464 
    -
    465 
    -    if output is None:
    466 
    -        click.echo(converted)
    467 
    -    else:
    468 
    -        outdir = os.path.dirname(output)
    469 
    -        os.makedirs(outdir, exist_ok=True)
    470 
    -        with open(output, 'wb') as f:
    471 
    -            f.write(converted.encode('utf-8'))
    454 
    +        run_session(description, tempdir, source_cache, palette, config_file, force)
    472 455
    473 456 
         return 0
    474 457

  • doc/sessions/developing.run
    ... ... @@ -16,7 +16,8 @@ commands:
    16 16
    17 17 
     # Apply a patch in the workspace
    18 18 
     - directory: ../examples/developing/
    19 
    -  command: show hello.bst; patch workspace_hello/files/src/hello.c update.patch;
    19 
    +  shell: True
    20 
    +  command: patch workspace_hello/hello.c update.patch
    20 21
    21 22 
     # Rebuild
    22 23 
     - directory: ../examples/developing/

  • doc/source/advanced-features/junction-elements.rst
    ... ... @@ -41,7 +41,7 @@ This element consists of a script which calls hello.bst's hello command.
    41 41 
     Building callHello.bst,
    42 42
    43 43 
     .. raw:: html
    44 
    -   :file: ../sessions-stored/junctions-build.html
    44 
    +   :file: ../sessions/junctions-build.html
    45 45
    46 46 
     You can see that the hello.bst element and it's dependencies from the autotools
    47 47 
     project have been build as part of the pipeline for callHello.bst.
    ... ... @@ -49,17 +49,18 @@ project have been build as part of the pipeline for callHello.bst.
    49 49 
     We can now invoke `bst shell`
    50 50
    51 51 
     .. raw:: html
    52 
    -   :file: ../sessions-stored/junctions-shell.html
    52 
    +   :file: ../sessions/junctions-shell.html
    53 53
    54 54 
     This runs the script files/callHello.sh which will makes use of the hello command from the hello.bst element in the autotools project.
    55 55
    56 
    +
    56 57 
     Cross-junction workspaces
    57 58 
     -------------------------
    58 59 
     You can open workspaces for elements in the project refered to by the junction
    59 60 
     using the syntax `bst open ${junction-name}:{element-name}`. In this example,
    60 61
    61 62 
     .. raw:: html
    62 
    -   :file: ../sessions-stored/junctions-workspace-open.html
    63 
    +   :file: ../sessions/junctions-workspace-open.html
    63 64
    64 65 
     This has opened a workspace for the hello.bst element from the autotools project.
    65 66 
     This workspace can now be used as normal.

  • doc/source/install_linux_distro.rst
    ... ... @@ -132,8 +132,8 @@ For the default plugins::
    132 132
    133 133 
     Ubuntu 16.04 LTS
    134 134 
     ^^^^^^^^^^^^^^^^
    135 
    -On Ubuntu 16.04, neither `bubblewrap<https://github.com/projectatomic/bubblewrap/>`
    136 
    -or `ostree<https://github.com/ostreedev/ostree>` are available in the official repositories.
    135 
    +On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    136 
    +or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    137 137 
     You will need to install them in whichever way you see fit. Refer the the upstream documentation
    138 138 
     for advice on this.
    139 139
    ... ... @@ -221,3 +221,15 @@ from `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_
    221 221 
     Alternatively, use
    222 222 
     `buildstream-git <https://aur.archlinux.org/packages/buildstream-git>`_
    223 223 
     for the lastest version of the development branch.
    224 
    +
    225 
    +Fedora
    226 
    +~~~~~~
    227 
    +
    228 
    +BuildStream is not yet in the official Fedora repositories, but you can
    229 
    +install it from a Copr:
    230 
    +
    231 
    +  sudo dnf copr enable bochecha/buildstream
    232 
    +  sudo dnf install buildstream
    233 
    +
    234 
    +Optionally, install the ``buildstream-docs`` package to have the BuildStream
    235 
    +documentation in Devhelp or GNOME Builder.

  • tests/sources/remote.py
    1 1 
     import os
    2 
    +import stat
    2 3 
     import pytest
    3 4
    4 5 
     from buildstream._exceptions import ErrorDomain
    ... ... @@ -82,7 +83,14 @@ def test_simple_file_build(cli, tmpdir, datafiles):
    82 83 
         result.assert_success()
    83 84 
         # Note that the url of the file in target.bst is actually /dir/file
    84 85 
         # but this tests confirms we take the basename
    85 
    -    assert(os.path.exists(os.path.join(checkoutdir, 'file')))
    86 
    +    checkout_file = os.path.join(checkoutdir, 'file')
    87 
    +    assert(os.path.exists(checkout_file))
    88 
    +
    89 
    +    mode = os.stat(checkout_file).st_mode
    90 
    +    # Assert not executable by anyone
    91 
    +    assert(not (mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)))
    92 
    +    # Assert not writeable by anyone other than me
    93 
    +    assert(not (mode & (stat.S_IWGRP | stat.S_IWOTH)))
    86 94
    87 95
    88 96 
     @pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
    ... ... @@ -119,6 +127,7 @@ def test_unique_key(cli, tmpdir, datafiles):
    119 127 
         generate_project(project, tmpdir)
    120 128 
         assert cli.get_element_state(project, 'target.bst') == "fetch needed"
    121 129 
         assert cli.get_element_state(project, 'target-custom.bst') == "fetch needed"
    130 
    +    assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
    122 131 
         # Try to fetch it
    123 132 
         result = cli.run(project=project, args=[
    124 133 
             'fetch', 'target.bst'
    ... ... @@ -127,7 +136,31 @@ def test_unique_key(cli, tmpdir, datafiles):
    127 136 
         # We should download the file only once
    128 137 
         assert cli.get_element_state(project, 'target.bst') == 'buildable'
    129 138 
         assert cli.get_element_state(project, 'target-custom.bst') == 'buildable'
    139 
    +    assert cli.get_element_state(project, 'target-custom-executable.bst') == 'buildable'
    130 140
    131 141 
         # But the cache key is different because the 'filename' is different.
    132 142 
         assert cli.get_element_key(project, 'target.bst') != \
    133 
    -        cli.get_element_key(project, 'target-custom.bst')
    143 
    +        cli.get_element_key(project, 'target-custom.bst') != \
    144 
    +        cli.get_element_key(project, 'target-custom-executable.bst')
    145 
    +
    146 
    +
    147 
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
    148 
    +def test_executable(cli, tmpdir, datafiles):
    149 
    +    '''This test confirms that the 'ecxecutable' parameter is honoured.
    150 
    +    '''
    151 
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    152 
    +    generate_project(project, tmpdir)
    153 
    +    checkoutdir = os.path.join(str(tmpdir), "checkout")
    154 
    +    assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
    155 
    +    # Try to fetch it
    156 
    +    result = cli.run(project=project, args=[
    157 
    +        'build', 'target-custom-executable.bst'
    158 
    +    ])
    159 
    +
    160 
    +    result = cli.run(project=project, args=[
    161 
    +        'checkout', 'target-custom-executable.bst', checkoutdir
    162 
    +    ])
    163 
    +    mode = os.stat(os.path.join(checkoutdir, 'some-custom-file')).st_mode
    164 
    +    assert (mode & stat.S_IEXEC)
    165 
    +    # Assert executable by anyone
    166 
    +    assert(mode & (stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH))

  • tests/sources/remote/unique-keys/target-custom-executable.bst
    1 
    +kind: import
    2 
    +description: test
    3 
    +sources:
    4 
    +- kind: remote
    5 
    +  url: tmpdir:/dir/file
    6 
    +  ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
    7 
    +  filename: some-custom-file
    8 
    +  executable: true

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]