GNOME.org
 

[Notes] [Git][BuildStream/buildstream][tpollard/591] 35 commits: setup.cfg: Show timing of the 20 slowest tests

Title: GitLab

Tom Pollard pushed to branch tpollard/591 at BuildStream / buildstream

Commits:

23 changed files:

Changes:

  • .gitignore
    ... ... @@ -25,7 +25,8 @@ __pycache__/
    25 25 
     # Generated version file
    26 26 
     buildstream/__version__.py
    27 27
    28 
    -#Autogenerated doc
    28 
    +# Autogenerated doc
    29 
    +doc/source/sessions/
    29 30 
     doc/source/elements/
    30 31 
     doc/source/sources/
    31 32 
     doc/source/modules.rst

  • .gitlab-ci.yml
    ... ... @@ -10,16 +10,6 @@ stages:
    10 10 
       - test
    11 11 
       - post
    12 12
    13 
    -# Avoid running all the tests post merge on
    14 
    -# master or on any release branch.
    15 
    -#
    16 
    -.tests-condition-template: &tests-condition
    17 
    -  only:
    18 
    -  - branches
    19 
    -  except:
    20 
    -  - master
    21 
    -  - /bst-1\..*/
    22 
    -
    23 13 
     #####################################################
    24 14 
     #                  Prepare stage                    #
    25 15 
     #####################################################
    ... ... @@ -96,26 +86,20 @@ source_dist:
    96 86 
     tests-debian-9:
    97 87 
       image: buildstream/testsuite-debian:9-master-114-4cab18e3
    98 88 
       <<: *linux-tests
    99 
    -  <<: *tests-condition
    100 89
    101 90 
     tests-fedora-27:
    102 91 
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    103 92 
       <<: *linux-tests
    104 
    -  <<: *tests-condition
    105 93
    106 94 
     tests-fedora-28:
    107 95 
       image: buildstream/testsuite-fedora:28-master-114-4cab18e3
    108 96 
       <<: *linux-tests
    109 
    -  <<: *tests-condition
    110 97
    111 98 
     tests-ubuntu-18.04:
    112 99 
       image: buildstream/testsuite-ubuntu:18.04-master-114-4cab18e3
    113 100 
       <<: *linux-tests
    114 
    -  <<: *tests-condition
    115 101
    116 102 
     tests-unix:
    117 
    -  <<: *tests-condition
    118 
    -
    119 103 
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    120 104 
       # can get rid of ostree - this is not possible with debian-8
    121 105 
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    ... ... @@ -154,15 +138,6 @@ tests-unix:
    154 138 
     # Note: We still do not enforce a consistent installation of python3-sphinx,
    155 139 
     #       as it will significantly grow the backing image.
    156 140 
     docs:
    157 
    -
    158 
    -  # Here we build the docs for every pre-merge CI, but avoid
    159 
    -  # the job on post-merge to stable branches, because we only
    160 
    -  # ever publish them from master
    161 
    -  only:
    162 
    -  - branches
    163 
    -  except:
    164 
    -  - /bst-1\..*/
    165 
    -
    166 141 
       stage: test
    167 142 
       script:
    168 143 
       - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
    ... ... @@ -178,19 +153,11 @@ docs:
    178 153 
         - public/
    179 154
    180 155
    181 
    -#####################################################
    182 
    -#                    Post stage                     #
    183 
    -#####################################################
    184 
    -
    185 
    -# Check code quality with codeclimate
    186 
    -# This needs some refactoring; we probably just want to provide the codeclimate.json directly
    187 
    -# as an output of radon, with some conversion
    156 
    +# Check code quality with gitlab's built-in feature.
    188 157 
     #
    189 
    -codequality:
    190 
    -  <<: *tests-condition
    191 
    -
    158 
    +code_quality:
    192 159 
       image: docker:stable
    193 
    -  stage: post
    160 
    +  stage: test
    194 161 
       variables:
    195 162 
         DOCKER_DRIVER: overlay2
    196 163 
       allow_failure: true
    ... ... @@ -204,11 +171,13 @@ codequality:
    204 171 
             --volume /var/run/docker.sock:/var/run/docker.sock
    205 172 
             "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
    206 173 
       artifacts:
    207 
    -    paths: [codeclimate.json]
    174 
    +    paths: [gl-code-quality-report.json]
    208 175
    209 
    -analysis:
    210 
    -  <<: *tests-condition
    176 
    +#####################################################
    177 
    +#                    Post stage                     #
    178 
    +#####################################################
    211 179
    180 
    +analysis:
    212 181 
       stage: post
    213 182 
       script:
    214 183 
       - |
    ... ... @@ -237,8 +206,6 @@ analysis:
    237 206 
     # Collate coverage reports
    238 207 
     #
    239 208 
     coverage:
    240 
    -  <<: *tests-condition
    241 
    -
    242 209 
       stage: post
    243 210 
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    244 211 
       script:

  • HACKING.rst
    ... ... @@ -390,6 +390,9 @@ Each *command* is a dictionary, the members of which are listed here:
    390 390
    391 391 
     * ``command``: The command to run, without the leading ``bst``
    392 392
    393 
    +* ``shell``: Specifying ``True`` indicates that ``command`` should be run as
    394 
    +  a shell command from the project directory, instead of a bst command (optional)
    395 
    +
    393 396 
     When adding a new ``.run`` file, one should normally also commit the new
    394 397 
     resulting generated ``.html`` file(s) into the ``doc/source/sessions-stored/``
    395 398 
     directory at the same time, this ensures that other developers do not need to

  • README.rst
    1 1 
     About
    2 2 
     -----
    3 
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/pipeline.svg
    4 
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    5 
    +
    6 
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
    7 
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    3 8
    4 9
    5 10 
     What is BuildStream?
    ... ... @@ -69,10 +74,10 @@ BuildStream operates on a set of YAML files (.bst files), as follows:
    69 74
    70 75 
     How can I get started?
    71 76 
     ======================
    72 
    -To start using BuildStream, first,
    73 
    -`install <https://buildstream.gitlab.io/buildstream/main_install.html>`_
    74 
    -BuildStream onto your machine and then follow our
    75 
    -`tutorial <https://buildstream.gitlab.io/buildstream/using_tutorial.html>`_.
    77 
    +To get started, first `install BuildStream by following the installation guide
    78 
    +<https://buildstream.gitlab.io/buildstream/main_install.html>`_
    79 
    +and then follow our tutorial in the
    80 
    +`user guide <https://buildstream.gitlab.io/buildstream/main_using.html>`_.
    76 81
    77 82 
     We also recommend exploring some existing BuildStream projects:
    78 83
    ... ... @@ -81,4 +86,3 @@ We also recommend exploring some existing BuildStream projects:
    81 86 
     * https://gitlab.com/baserock/definitions
    82 87
    83 88 
     If you have any questions please ask on our `#buildstream <irc://irc.gnome.org/buildstream>`_ channel in `irc.gnome.org <irc://irc.gnome.org>`_
    84 
    -

  • buildstream/_project.py
    ... ... @@ -398,6 +398,11 @@ class Project():
    398 398 
                     "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
    399 399 
                     .format(format_version, major, minor, BST_FORMAT_VERSION))
    400 400
    401 
    +        # Check if project.conf contains name key field
    402 
    +        if self._project_conf.get('name') is None:
    403 
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    404 
    +                            "{}: project.conf does not contain expected key '{}'".format(projectfile, 'name'))
    405 
    +
    401 406 
             # The project name, element path and option declarations
    402 407 
             # are constant and cannot be overridden by option conditional statements
    403 408 
             self.name = _yaml.node_get(pre_config_node, str, 'name')

  • buildstream/buildelement.py
    ... ... @@ -233,14 +233,13 @@ class BuildElement(Element):
    233 233 
             return commands
    234 234
    235 235 
         def __run_command(self, sandbox, cmd, cmd_name):
    236 
    -        with self.timed_activity("Running {}".format(cmd_name)):
    237 
    -            self.status("Running {}".format(cmd_name), detail=cmd)
    238 
    -
    239 
    -            # Note the -e switch to 'sh' means to exit with an error
    240 
    -            # if any untested command fails.
    241 
    -            #
    242 
    -            exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    243 
    -                                   SandboxFlags.ROOT_READ_ONLY)
    244 
    -            if exitcode != 0:
    245 
    -                raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    246 
    -                                   collect=self.get_variable('install-root'))
    236 
    +        self.status("Running {}".format(cmd_name), detail=cmd)
    237 
    +
    238 
    +        # Note the -e switch to 'sh' means to exit with an error
    239 
    +        # if any untested command fails.
    240 
    +        #
    241 
    +        exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    242 
    +                               SandboxFlags.ROOT_READ_ONLY)
    243 
    +        if exitcode != 0:
    244 
    +            raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    245 
    +                               collect=self.get_variable('install-root'))

  • buildstream/element.py
    ... ... @@ -1101,9 +1101,12 @@ class Element(Plugin):
    1101 1101 
                 # until the full cache query below.
    1102 1102 
                 if (not self.__assemble_scheduled and not self.__assemble_done and
    1103 1103 
                         not self.__cached_success(keystrength=_KeyStrength.WEAK) and
    1104 
    -                    not self._pull_pending() and self._is_required()):
    1105 
    -                self._schedule_assemble()
    1106 
    -                return
    1104 
    +                    not self._pull_pending()):
    1105 
    +                # For uncached workspaced elements, assemble is required
    1106 
    +                # even if we only need the cache key
    1107 
    +                if self._is_required() or self._get_workspace():
    1108 
    +                    self._schedule_assemble()
    1109 
    +                    return
    1107 1110
    1108 1111 
             if self.__strict_cache_key is None:
    1109 1112 
                 dependencies = [
    ... ... @@ -1126,13 +1129,17 @@ class Element(Plugin):
    1126 1129 
                     self.__weak_cached = self.__artifacts.contains(self, self.__weak_cache_key)
    1127 1130
    1128 1131 
             if (not self.__assemble_scheduled and not self.__assemble_done and
    1129 
    -                not self._cached_success() and not self._pull_pending() and self._is_required()):
    1132 
    +                not self._cached_success() and not self._pull_pending()):
    1130 1133 
                 # Workspaced sources are considered unstable if a build is pending
    1131 1134 
                 # as the build will modify the contents of the workspace.
    1132 1135 
                 # Determine as early as possible if a build is pending to discard
    1133 1136 
                 # unstable cache keys.
    1134 
    -            self._schedule_assemble()
    1135 
    -            return
    1137 
    +
    1138 
    +            # For uncached workspaced elements, assemble is required
    1139 
    +            # even if we only need the cache key
    1140 
    +            if self._is_required() or self._get_workspace():
    1141 
    +                self._schedule_assemble()
    1142 
    +                return
    1136 1143
    1137 1144 
             if self.__cache_key is None:
    1138 1145 
                 # Calculate strong cache key
    ... ... @@ -1430,7 +1437,6 @@ class Element(Plugin):
    1430 1437 
         # in a subprocess.
    1431 1438 
         #
    1432 1439 
         def _schedule_assemble(self):
    1433 
    -        assert self._is_required()
    1434 1440 
             assert not self.__assemble_scheduled
    1435 1441 
             self.__assemble_scheduled = True
    1436 1442
    ... ... @@ -1438,6 +1444,8 @@ class Element(Plugin):
    1438 1444 
             for dep in self.dependencies(Scope.BUILD, recurse=False):
    1439 1445 
                 dep._set_required()
    1440 1446
    1447 
    +        self._set_required()
    1448 
    +
    1441 1449 
             # Invalidate workspace key as the build modifies the workspace directory
    1442 1450 
             workspace = self._get_workspace()
    1443 1451 
             if workspace:
    ... ... @@ -1661,6 +1669,10 @@ class Element(Plugin):
    1661 1669 
         #   (bool): Whether a pull operation is pending
    1662 1670 
         #
    1663 1671 
         def _pull_pending(self):
    1672 
    +        if self._get_workspace():
    1673 
    +            # Workspace builds are never pushed to artifact servers
    1674 
    +            return False
    1675 
    +
    1664 1676 
             if self.__strong_cached:
    1665 1677 
                 # Artifact already in local cache
    1666 1678 
                 return False

  • doc/bst2html.py
    ... ... @@ -29,7 +29,7 @@ import sys
    29 29 
     import re
    30 30 
     import shlex
    31 31 
     import subprocess
    32 
    -from collections import Mapping
    32 
    +from collections.abc import Mapping
    33 33 
     from contextlib import contextmanager
    34 34 
     from tempfile import TemporaryDirectory
    35 35

  • doc/source/developing/workspaces.rst
    ... ... @@ -4,9 +4,9 @@
    4 4
    5 5 
     Workspaces
    6 6 
     ==========
    7 
    -
    8 7 
     In this section we will cover the use of BuildStream's workspaces feature when devloping on a
    9 8 
     BuildStream project.
    9 
    +
    10 10 
     .. note::
    11 11
    12 12 
        This example is distributed with BuildStream
    ... ... @@ -25,7 +25,6 @@ make changes to the source code of Buildstream elements by making use of
    25 25 
     BuildStream's workspace command.
    26 26
    27 27
    28 
    -
    29 28 
     Opening a workspace
    30 29 
     -------------------
    31 30 
     First we need to open a workspace, we can do this by running
    ... ... @@ -37,12 +36,12 @@ This command has created the workspace_hello directory in which you can see
    37 36 
     the source for the hello.bst element, i.e. hello.c and the corresponding
    38 37 
     makefile.
    39 38
    40 
    -
    41 39 
     You can view existing workspaces using
    42 40
    43 41 
     .. raw:: html
    44 42 
        :file: ../sessions/developing-workspace-list.html
    45 43
    44 
    +
    46 45 
     Making code changes
    47 46 
     -------------------
    48 47 
     Let's say we want to alter the message printed when the hello command is run.
    ... ... @@ -66,6 +65,7 @@ This gives us the new message we changed in hello.c.
    66 65 
     From this point we have several options. If the source is under version control
    67 66 
     we can commit our changes and push them to the remote repository.
    68 67
    68 
    +
    69 69 
     Closing your workspace
    70 70 
     ----------------------
    71 71 
     If we want to close the workspace and come back to our changes later, we can

  • doc/source/install_linux_distro.rst
    1 1
    2 
    -.. _install:
    3 
    -
    4 
    -Installing BuildStream on a Linux distro
    5 
    -========================================
    6 
    -BuildStream requires the following base system requirements:
    7 
    -
    8 
    -* python3 >= 3.5
    9 
    -* bubblewrap >= 0.1.2
    10 
    -* fuse2
    11 
    -
    12 
    -BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    13 
    -Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    14 
    -requirements of specific plugins.
    15 
    -
    16 
    -The default plugins with extra host dependencies are:
    17 
    -
    18 
    -* bzr
    19 
    -* deb
    20 
    -* git
    21 
    -* ostree
    22 
    -* patch
    23 
    -* tar
    24 
    -
    25 
    -If you intend to push built artifacts to a remote artifact server,
    26 
    -which requires special permissions, you will also need:
    27 
    -
    28 
    -* ssh
    29 
    -
    30 
    -
    31 
    -Installing from source (recommended)
    32 
    -------------------------------------
    33 
    -Until BuildStream is available in your distro, you will need to install
    34 
    -it yourself from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    35 
    -using python's ``pip`` package manager.
    36 
    -
    37 
    -For the purpose of installing BuildStream while there are no distro packages,
    38 
    -you will additionally need:
    39 
    -
    40 
    -* pip for python3 (only required for setup)
    41 
    -* Python 3 development libraries and headers
    42 
    -* git (to checkout buildstream)
    43 
    -
    44 
    -
    45 
    -Installing dependencies
    46 
    -~~~~~~~~~~~~~~~~~~~~~~~
    47 
    -
    48 
    -
    49 
    -Arch Linux
    50 
    -++++++++++
    51 
    -Install the dependencies with::
    52 
    -
    53 
    -  sudo pacman -S \
    54 
    -      python fuse2 bubblewrap \
    55 
    -      python-pip git
    56 
    -
    57 
    -For the default plugins::
    58 
    -
    59 
    -  sudo pacman -S \
    60 
    -      bzr git lzip ostree patch python-gobject
    61 
    -
    62 
    -
    63 
    -The package *python-arpy* is required by the deb source plugin. This is not
    64 
    -obtainable via `pacman`, you must get *python-arpy* from AUR:
    65 
    -https://aur.archlinux.org/packages/python-arpy/
    66 
    -
    67 
    -To install::
    68 
    -
    69 
    -  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    70 
    -  tar -xvf python-arpy.tar.gz
    71 
    -  cd python-arpy
    72 
    -  makepkg -si
    73 
    -
    74 
    -Debian
    75 
    -++++++
    76 
    -Install the dependencies with::
    77 
    -
    78 
    -  sudo apt-get install \
    79 
    -      python3 fuse bubblewrap \
    80 
    -      python3-pip python3-dev git
    81 
    -
    82 
    -For the default plugins:
    83 
    -
    84 
    -Stretch
    85 
    -^^^^^^^
    86 
    -With stretch, you first need to ensure that you have the backports repository
    87 
    -setup as described `here <https://backports.debian.org/Instructions/>`_
    88 
    -
    89 
    -By adding the following line to your sources.list::
    90 
    -
    91 
    -  deb http://deb.debian.org/debian stretch-backports main
    92 
    -
    93 
    -And then running::
    94 
    -
    95 
    -  sudo apt update
    96 
    -
    97 
    -At this point you should be able to get the system requirements for the default plugins with::
    98 
    -
    99 
    -  sudo apt install \
    100 
    -      bzr git lzip patch python3-arpy python3-gi
    101 
    -  sudo apt install -t stretch-backports \
    102 
    -      gir1.2-ostree-1.0 ostree
    103 
    -
    104 
    -Buster or Sid
    105 
    -^^^^^^^^^^^^^
    106 
    -For debian unstable or testing, only the following line should be enough
    107 
    -to get the system requirements for the default plugins installed::
    108 
    -
    109 
    -  sudo apt-get install \
    110 
    -      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    111 
    -
    112 
    -
    113 
    -Fedora
    114 
    -++++++
    115 
    -For recent fedora systems, the following line should get you the system
    116 
    -requirements you need::
    117 
    -
    118 
    -  dnf install -y \
    119 
    -      python3 fuse bubblewrap \
    120 
    -      python3-pip python3-devel git
    121 
    -
    122 
    -For the default plugins::
    123 
    -
    124 
    -  dnf install -y \
    125 
    -      bzr git lzip patch ostree python3-gobject
    126 
    -  pip3 install --user arpy
    127 
    -
    128 
    -
    129 
    -Ubuntu
    130 
    -++++++
    131 
    -
    132 
    -Ubuntu 18.04 LTS or later
    133 
    -^^^^^^^^^^^^^^^^^^^^^^^^^
    134 
    -Install the dependencies with::
    135 
    -
    136 
    -  sudo apt install \
    137 
    -      python3 fuse bubblewrap \
    138 
    -      python3-pip python3-dev git
    139 
    -
    140 
    -For the default plugins::
    141 
    -
    142 
    -  sudo apt install \
    143 
    -      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    144 
    -
    145 
    -Ubuntu 16.04 LTS
    146 
    -^^^^^^^^^^^^^^^^
    147 
    -On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    148 
    -or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    149 
    -You will need to install them in whichever way you see fit. Refer the the upstream documentation
    150 
    -for advice on this.
    151 
    -
    152 
    -
    153 
    -Installing
    154 
    -~~~~~~~~~~
    155 
    -Once you have the base system dependencies, you can install the BuildStream
    156 
    -python package as a regular user.
    157 
    -
    158 
    -Via PyPI (recommended)
    159 
    -++++++++++++++++++++++
    160 
    -::
    161 
    -
    162 
    -  pip3 install --user BuildStream
    163 
    -
    164 
    -This will install latest stable version of BuildStream and its pure python
    165 
    -dependencies into your user's homedir in ``~/.local``.
    166 
    -
    167 
    -Keep following the instructions below to ensure that the ``bst``
    168 
    -command is in your ``PATH`` and to enable bash completions for it.
    169 
    -
    170 
    -.. note::
    171 
    -
    172 
    -  If you want a specific version of BuildStream, you can install it using
    173 
    -  ``pip install --user BuildStream==<version-number>``
    174 
    -
    175 
    -Via Git checkout
    176 
    -++++++++++++++++
    177 
    -::
    178 
    -
    179 
    -  git clone https://gitlab.com/BuildStream/buildstream.git
    180 
    -  cd buildstream
    181 
    -  pip3 install --user -e .
    182 
    -
    183 
    -This will install buildstream's pure python dependencies into
    184 
    -your user's homedir in ``~/.local`` and will run BuildStream directly
    185 
    -from the git checkout directory.
    186 
    -
    187 
    -Keep following the instructions below to ensure that the ``bst``
    188 
    -command is in your ``PATH`` and to enable bash completions for it.
    189 
    -
    190 
    -.. note::
    191 
    -
    192 
    -   We recommend the ``-e`` option because you can upgrade your
    193 
    -   installation by simply updating the checked out git repository.
    194 
    -
    195 
    -   If you want a full installation that is not linked to your
    196 
    -   git checkout, just omit the ``-e`` option from the above commands.
    197 
    -
    198 
    -
    199 
    -Adjust PATH
    200 
    -~~~~~~~~~~~
    201 
    -Since BuildStream is now installed under your local user's install directories,
    202 
    -you need to ensure that ``PATH`` is adjusted.
    203 
    -
    204 
    -A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    205 
    -
    206 
    -  export PATH="${PATH}:${HOME}/.local/bin"
    207 
    -
    208 
    -.. note::
    209 
    -
    210 
    -   You will have to restart your terminal in order for these changes to take effect.
    211 
    -
    212 
    -
    213 
    -Bash completions
    214 
    -~~~~~~~~~~~~~~~~
    215 
    -Bash completions are supported by sourcing the ``buildstream/data/bst``
    216 
    -script found in the BuildStream repository. On many systems this script
    217 
    -can be installed into a completions directory but when installing BuildStream
    218 
    -without a package manager this is not an option.
    219 
    -
    220 
    -To enable completions for an installation of BuildStream you
    221 
    -installed yourself from git, just append the script verbatim
    222 
    -to your ``~/.bash_completion``:
    223 
    -
    224 
    -.. literalinclude:: ../../buildstream/data/bst
    225 
    -   :language: yaml
    226 
    -
    227 
    -
    228 
    -Upgrading BuildStream
    229 
    -~~~~~~~~~~~~~~~~~~~~~
    230 
    -
    231 
    -Via PyPI
    232 
    -++++++++
    233 
    -
    234 
    -If you installed BuildStream from PyPI, you can update it like so::
    235 
    -
    236 
    -  pip install --user --upgrade BuildStream
    237 
    -
    238 
    -Via Git checkout
    239 
    -++++++++++++++++
    240 
    -
    241 
    -If you installed BuildStream from a local git checkout using ``-e`` option, all
    242 
    -you need to do to upgrade BuildStream is to update your local git checkout::
    243 
    -
    244 
    -  cd /path/to/buildstream
    245 
    -  git pull --rebase
    246 
    -
    247 
    -If you did not specify the ``-e`` option at install time or the dependancies
    248 
    -have changed, you will need to cleanly reinstall BuildStream::
    249 
    -
    250 
    -  pip3 uninstall buildstream
    251 
    -  cd /path/to/buildstream
    252 
    -  git pull --rebase
    253 
    -  pip3 install --user .
    254 2
    3 
    +.. _install_linux_distro:
    255 4
    256 5 
     Installing from distro packages
    257 
    --------------------------------
    6 
    +===============================
    7 
    +BuildStream is available on some linux distributions, here are
    8 
    +some install instructions for the linux distributions which
    9 
    +have packaged BuildStream.
    258 10
    259 11
    260 12 
     Arch Linux
    261 
    -~~~~~~~~~~
    13 
    +----------
    262 14 
     Packages for Arch exist in `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_packages>`_.
    263 15 
     Two different package versions are available:
    264 16
    ... ... @@ -267,8 +19,7 @@ Two different package versions are available:
    267 19
    268 20
    269 21 
     Fedora
    270 
    -~~~~~~
    271 
    -
    22 
    +------
    272 23 
     BuildStream is not yet in the official Fedora repositories, but you can
    273 24 
     install it from a Copr::
    274 25

  • doc/source/install_source.rst
    1 
    +
    2 
    +
    3 
    +Installing from source
    4 
    +======================
    5 
    +Until BuildStream is available in :ref:`your distro <install_linux_distro>`, you will
    6 
    +need to install it yourself from source.
    7 
    +
    8 
    +
    9 
    +Installing dependencies
    10 
    +-----------------------
    11 
    +Before installing BuildStream from source, it is necessary to first install
    12 
    +the system dependencies. Below are some linux distribution specific instructions
    13 
    +for installing these dependencies.
    14 
    +
    15 
    +BuildStream requires the following base system requirements:
    16 
    +
    17 
    +* python3 >= 3.5
    18 
    +* bubblewrap >= 0.1.2
    19 
    +* fuse2
    20 
    +
    21 
    +BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    22 
    +Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    23 
    +requirements of specific plugins.
    24 
    +
    25 
    +The default plugins with extra host dependencies are:
    26 
    +
    27 
    +* bzr
    28 
    +* deb
    29 
    +* git
    30 
    +* ostree
    31 
    +* patch
    32 
    +* tar
    33 
    +
    34 
    +If you intend to push built artifacts to a remote artifact server,
    35 
    +which requires special permissions, you will also need:
    36 
    +
    37 
    +* ssh
    38 
    +
    39 
    +
    40 
    +Arch Linux
    41 
    +~~~~~~~~~~
    42 
    +Install the dependencies with::
    43 
    +
    44 
    +  sudo pacman -S \
    45 
    +      python fuse2 bubblewrap \
    46 
    +      python-pip
    47 
    +
    48 
    +For the default plugins::
    49 
    +
    50 
    +  sudo pacman -S \
    51 
    +      bzr git lzip ostree patch python-gobject
    52 
    +
    53 
    +
    54 
    +The package *python-arpy* is required by the deb source plugin. This is not
    55 
    +obtainable via `pacman`, you must get *python-arpy* from AUR:
    56 
    +https://aur.archlinux.org/packages/python-arpy/
    57 
    +
    58 
    +To install::
    59 
    +
    60 
    +  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    61 
    +  tar -xvf python-arpy.tar.gz
    62 
    +  cd python-arpy
    63 
    +  makepkg -si
    64 
    +
    65 
    +
    66 
    +Debian
    67 
    +~~~~~~
    68 
    +Install the dependencies with::
    69 
    +
    70 
    +  sudo apt-get install \
    71 
    +      python3 fuse bubblewrap \
    72 
    +      python3-pip python3-dev
    73 
    +
    74 
    +For the default plugins:
    75 
    +
    76 
    +
    77 
    +Stretch
    78 
    ++++++++
    79 
    +With stretch, you first need to ensure that you have the backports repository
    80 
    +setup as described `here <https://backports.debian.org/Instructions/>`_
    81 
    +
    82 
    +By adding the following line to your sources.list::
    83 
    +
    84 
    +  deb http://deb.debian.org/debian stretch-backports main
    85 
    +
    86 
    +And then running::
    87 
    +
    88 
    +  sudo apt update
    89 
    +
    90 
    +At this point you should be able to get the system requirements for the default plugins with::
    91 
    +
    92 
    +  sudo apt install \
    93 
    +      bzr git lzip patch python3-arpy python3-gi
    94 
    +  sudo apt install -t stretch-backports \
    95 
    +      gir1.2-ostree-1.0 ostree
    96 
    +
    97 
    +
    98 
    +Buster or Sid
    99 
    ++++++++++++++
    100 
    +For debian unstable or testing, only the following line should be enough
    101 
    +to get the system requirements for the default plugins installed::
    102 
    +
    103 
    +  sudo apt-get install \
    104 
    +      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    105 
    +
    106 
    +
    107 
    +Fedora
    108 
    +~~~~~~
    109 
    +For recent fedora systems, the following line should get you the system
    110 
    +requirements you need::
    111 
    +
    112 
    +  dnf install -y \
    113 
    +      python3 fuse bubblewrap \
    114 
    +      python3-pip python3-devel
    115 
    +
    116 
    +For the default plugins::
    117 
    +
    118 
    +  dnf install -y \
    119 
    +      bzr git lzip patch ostree python3-gobject
    120 
    +  pip3 install --user arpy
    121 
    +
    122 
    +
    123 
    +Ubuntu
    124 
    +~~~~~~
    125 
    +
    126 
    +
    127 
    +Ubuntu 18.04 LTS or later
    128 
    ++++++++++++++++++++++++++
    129 
    +Install the dependencies with::
    130 
    +
    131 
    +  sudo apt install \
    132 
    +      python3 fuse bubblewrap \
    133 
    +      python3-pip python3-dev
    134 
    +
    135 
    +For the default plugins::
    136 
    +
    137 
    +  sudo apt install \
    138 
    +      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    139 
    +
    140 
    +
    141 
    +Ubuntu 16.04 LTS
    142 
    +++++++++++++++++
    143 
    +On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    144 
    +or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    145 
    +You will need to install them in whichever way you see fit. Refer the the upstream documentation
    146 
    +for advice on this.
    147 
    +
    148 
    +
    149 
    +Installing
    150 
    +----------
    151 
    +Once you have the base system dependencies, you can install the BuildStream
    152 
    +python package as a regular user.
    153 
    +
    154 
    +
    155 
    +Installing from PyPI (recommended)
    156 
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    157 
    +Since we only ever publish :ref:`release versions <install_semantic_versioning>` on
    158 
    +PyPI, it is currently recommended to use this installation path. This will
    159 
    +ensure that you always have the latest recommended version of BuildStream that
    160 
    +we recommend.
    161 
    +
    162 
    +To install from PyPI, you will additionally require:
    163 
    +
    164 
    +* pip for python3 (only required for setup)
    165 
    +* Python 3 development libraries and headers
    166 
    +
    167 
    +Simply run the following command::
    168 
    +
    169 
    +  pip3 install --user BuildStream
    170 
    +
    171 
    +This will install latest stable version of BuildStream and its pure python
    172 
    +dependencies into your user's homedir in ``~/.local``.
    173 
    +
    174 
    +Keep following the instructions below to ensure that the ``bst``
    175 
    +command is in your ``PATH`` and to enable bash completions for it.
    176 
    +
    177 
    +.. note::
    178 
    +
    179 
    +  If you want a specific version of BuildStream, you can install it using
    180 
    +  ``pip install --user BuildStream==<version-number>``
    181 
    +
    182 
    +
    183 
    +Upgrading from PyPI
    184 
    ++++++++++++++++++++
    185 
    +Once you have already installed BuildStream from PyPI, you can later update
    186 
    +to the latest recommended version like so::
    187 
    +
    188 
    +  pip install --user --upgrade BuildStream
    189 
    +
    190 
    +
    191 
    +.. _install_git_checkout:
    192 
    +
    193 
    +Installing from a git checkout
    194 
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    195 
    +To install directly from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    196 
    +using python's ``pip`` package manager, you will additionally require:
    197 
    +
    198 
    +* pip for python3 (only required for setup)
    199 
    +* Python 3 development libraries and headers
    200 
    +* git (to checkout BuildStream)
    201 
    +
    202 
    +Before installing, please check the existing tags in the git repository
    203 
    +and determine which version you want to install, and whether you want
    204 
    +to install an official release version (recommended), or a development snapshot
    205 
    +to help us out testing the bleeding edge of development. Follow the
    206 
    +:ref:`semantic versioning guide <install_semantic_versioning>` to determine
    207 
    +which tag you intend to install.
    208 
    +
    209 
    +Run the following commands::
    210 
    +
    211 
    +  git clone https://gitlab.com/BuildStream/buildstream.git
    212 
    +  cd buildstream
    213 
    +  git checkout <desired release tag>
    214 
    +  pip3 install --user -e .
    215 
    +
    216 
    +This will install buildstream's pure python dependencies into
    217 
    +your user's homedir in ``~/.local`` and will run BuildStream directly
    218 
    +from the git checkout directory.
    219 
    +
    220 
    +Keep following the instructions below to ensure that the ``bst``
    221 
    +command is in your ``PATH`` and to enable bash completions for it.
    222 
    +
    223 
    +.. note::
    224 
    +
    225 
    +   We recommend the ``-e`` option because you can upgrade your
    226 
    +   installation by simply updating the checked out git repository.
    227 
    +
    228 
    +   If you want a full installation that is not linked to your
    229 
    +   git checkout, just omit the ``-e`` option from the above commands.
    230 
    +
    231 
    +
    232 
    +Upgrading from a git checkout
    233 
    ++++++++++++++++++++++++++++++
    234 
    +If you installed BuildStream from a local git checkout using ``-e`` option, all
    235 
    +you need to do to upgrade BuildStream is to update your local git checkout::
    236 
    +
    237 
    +  cd /path/to/buildstream
    238 
    +  git pull --rebase
    239 
    +
    240 
    +If you did not specify the ``-e`` option at install time or the dependancies
    241 
    +have changed, you will need to cleanly reinstall BuildStream::
    242 
    +
    243 
    +  pip3 uninstall buildstream
    244 
    +  cd /path/to/buildstream
    245 
    +  git pull --rebase
    246 
    +  pip3 install --user .
    247 
    +
    248 
    +.. note::
    249 
    +
    250 
    +   If BuildStream has added any dependencies since the last upgrade,
    251 
    +   you will need to uninstall and reinstall to ensure those dependencies
    252 
    +   are met, regardless of whether you have used the ``-e`` option at
    253 
    +   install time.
    254 
    +
    255 
    +
    256 
    +Post install setup
    257 
    +------------------
    258 
    +After having installed from source using any of the above methods, some
    259 
    +setup will be required to use BuildStream.
    260 
    +
    261 
    +
    262 
    +Adjust PATH
    263 
    +~~~~~~~~~~~
    264 
    +Since BuildStream is now installed under your local user's install directories,
    265 
    +you need to ensure that ``PATH`` is adjusted.
    266 
    +
    267 
    +A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    268 
    +
    269 
    +  export PATH="${PATH}:${HOME}/.local/bin"
    270 
    +
    271 
    +.. note::
    272 
    +
    273 
    +   You will have to restart your terminal in order for these changes to take effect.
    274 
    +
    275 
    +
    276 
    +Bash completions
    277 
    +~~~~~~~~~~~~~~~~
    278 
    +Bash completions are supported by sourcing the ``buildstream/data/bst``
    279 
    +script found in the BuildStream repository. On many systems this script
    280 
    +can be installed into a completions directory but when installing BuildStream
    281 
    +without a package manager this is not an option.
    282 
    +
    283 
    +To enable completions for an installation of BuildStream you
    284 
    +installed yourself from git, just append the script verbatim
    285 
    +to your ``~/.bash_completion``:
    286 
    +
    287 
    +.. literalinclude:: ../../buildstream/data/bst
    288 
    +   :language: yaml

  • doc/source/install_versions.rst
    1 
    +
    2 
    +
    3 
    +.. _install_semantic_versioning:
    4 
    +
    5 
    +Semantic Versioning
    6 
    +===================
    7 
    +BuildStream follows the Semantic Versioning Convention `(SemVer) <https://semver.org/>`_,
    8 
    +and uses even minor point numbers to denote releases intended for users while
    9 
    +odd minor point numbers represent development snapshops.
    10 
    +
    11 
    +For example, for a given version number ``X.Y.Z``
    12 
    + * The ``X.<even number>.*`` versions are releases intended for users.
    13 
    + * The ``X.<odd number>.*`` versions are development spanshots intended for testing.
    14 
    +
    15 
    +If you are :ref:`installing from git <install_git_checkout>`, please look for the latest
    16 
    +tag in the latest release branch to ensure you're getting the latest release.
    17 
    +
    18 
    +Current release branches:
    19 
    + * `bst-1.2 (latest) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.2>`_
    20 
    + * `bst-1.0 (deprecated) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.0>`_

  • doc/source/main_install.rst
    1 
    +
    2 
    +
    3 
    +.. _install:
    4 
    +
    1 5 
     Install
    2 6 
     =======
    3 
    -This section covers how to install BuildStream onto your machine, how to run
    4 
    -BuildStream inside a docker image and also how to configure an artifact server.
    7 
    +This section provides instructions for installing BuildStream and its
    8 
    +companion artifact server on various platforms, along with any installation
    9 
    +related materials.
    5 10
    6 11 
     .. note::
    7 12
    8 
    -   BuildStream is not currently supported natively on macOS and Windows. Windows
    9 
    -   and macOS users should refer to :ref:`docker`.
    13 
    +   BuildStream is currently only supported natively on Linux. Users of Unix-like
    14 
    +   systems where Docker is available can still use BuildStream by following the
    15 
    +   :ref:`Docker install guide <docker>`
    10 16
    11 17 
     .. toctree::
    12 
    -   :maxdepth: 2
    18 
    +   :maxdepth: 1
    13 19
    20 
    +   install_source
    14 21 
        install_linux_distro
    15 22 
        install_docker
    16 23 
        install_artifacts
    24 
    +   install_versions

  • doc/source/sessions/developing-build-after-changes.html → doc/source/sessions-stored/developing-build-after-changes.html

  • doc/source/sessions/developing-close-workspace.html → doc/source/sessions-stored/developing-close-workspace.html

  • doc/source/sessions/developing-discard-workspace.html → doc/source/sessions-stored/developing-discard-workspace.html

  • doc/source/sessions/developing-reopen-workspace.html → doc/source/sessions-stored/developing-reopen-workspace.html

  • doc/source/sessions/developing-reset-workspace.html → doc/source/sessions-stored/developing-reset-workspace.html

  • doc/source/sessions/developing-shell-after-changes.html → doc/source/sessions-stored/developing-shell-after-changes.html

  • doc/source/sessions/developing-workspace-list.html → doc/source/sessions-stored/developing-workspace-list.html

  • doc/source/sessions/developing-workspace-open.html → doc/source/sessions-stored/developing-workspace-open.html

  • setup.cfg
    ... ... @@ -11,7 +11,7 @@ parentdir_prefix = BuildStream-
    11 11 
     test=pytest
    12 12
    13 13 
     [tool:pytest]
    14 
    -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc
    14 
    +addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
    15 15 
     norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
    16 16 
     python_files = tests/*/*.py
    17 17 
     pep8maxlinelength = 119

  • tests/frontend/workspace.py
    ... ... @@ -780,3 +780,73 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
    780 780 
             'build', element_name
    781 781 
         ])
    782 782 
         result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced")
    783 
    +
    784 
    +
    785 
    +@pytest.mark.datafiles(DATA_DIR)
    786 
    +@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
    787 
    +def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
    788 
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    789 
    +    element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
    790 
    +                                                      datafiles, 'git', False)
    791 
    +
    792 
    +    element_path = os.path.join(project, 'elements')
    793 
    +    back_dep_element_name = 'workspace-test-back-dep.bst'
    794 
    +
    795 
    +    # Write out our test target
    796 
    +    element = {
    797 
    +        'kind': 'compose',
    798 
    +        'depends': [
    799 
    +            {
    800 
    +                'filename': element_name,
    801 
    +                'type': 'build'
    802 
    +            }
    803 
    +        ]
    804 
    +    }
    805 
    +    _yaml.dump(element,
    806 
    +               os.path.join(element_path,
    807 
    +                            back_dep_element_name))
    808 
    +
    809 
    +    # Modify workspace
    810 
    +    shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
    811 
    +    os.makedirs(os.path.join(workspace, 'etc'))
    812 
    +    with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
    813 
    +        f.write("PONY='pink'")
    814 
    +
    815 
    +    # Configure strict mode
    816 
    +    strict_mode = True
    817 
    +    if strict != 'strict':
    818 
    +        strict_mode = False
    819 
    +    cli.configure({
    820 
    +        'projects': {
    821 
    +            'test': {
    822 
    +                'strict': strict_mode
    823 
    +            }
    824 
    +        }
    825 
    +    })
    826 
    +
    827 
    +    # Build artifact with dependency's modified workspace
    828 
    +    assert cli.get_element_state(project, element_name) == 'buildable'
    829 
    +    assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
    830 
    +    assert cli.get_element_state(project, back_dep_element_name) == 'waiting'
    831 
    +    assert cli.get_element_key(project, back_dep_element_name) == "{:?<64}".format('')
    832 
    +    result = cli.run(project=project, args=['build', back_dep_element_name])
    833 
    +    result.assert_success()
    834 
    +    assert cli.get_element_state(project, element_name) == 'cached'
    835 
    +    assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
    836 
    +    assert cli.get_element_state(project, back_dep_element_name) == 'cached'
    837 
    +    assert cli.get_element_key(project, back_dep_element_name) != "{:?<64}".format('')
    838 
    +    result = cli.run(project=project, args=['build', back_dep_element_name])
    839 
    +    result.assert_success()
    840 
    +
    841 
    +    # Checkout the result
    842 
    +    result = cli.run(project=project, args=[
    843 
    +        'checkout', back_dep_element_name, checkout
    844 
    +    ])
    845 
    +    result.assert_success()
    846 
    +
    847 
    +    # Check that the pony.conf from the modified workspace exists
    848 
    +    filename = os.path.join(checkout, 'etc', 'pony.conf')
    849 
    +    assert os.path.exists(filename)
    850 
    +
    851 
    +    # Check that the original /usr/bin/hello is not in the checkout
    852 
    +    assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]