[Notes] [Git][BuildStream/buildstream][tpollard/591] 35 commits: setup.cfg: Show timing of the 20 slowest tests



Title: GitLab

Tom Pollard pushed to branch tpollard/591 at BuildStream / buildstream

Commits:

23 changed files:

Changes:

  • .gitignore
    ... ... @@ -25,7 +25,8 @@ __pycache__/
    25 25
     # Generated version file
    
    26 26
     buildstream/__version__.py
    
    27 27
     
    
    28
    -#Autogenerated doc
    
    28
    +# Autogenerated doc
    
    29
    +doc/source/sessions/
    
    29 30
     doc/source/elements/
    
    30 31
     doc/source/sources/
    
    31 32
     doc/source/modules.rst
    

  • .gitlab-ci.yml
    ... ... @@ -10,16 +10,6 @@ stages:
    10 10
       - test
    
    11 11
       - post
    
    12 12
     
    
    13
    -# Avoid running all the tests post merge on
    
    14
    -# master or on any release branch.
    
    15
    -#
    
    16
    -.tests-condition-template: &tests-condition
    
    17
    -  only:
    
    18
    -  - branches
    
    19
    -  except:
    
    20
    -  - master
    
    21
    -  - /bst-1\..*/
    
    22
    -
    
    23 13
     #####################################################
    
    24 14
     #                  Prepare stage                    #
    
    25 15
     #####################################################
    
    ... ... @@ -96,26 +86,20 @@ source_dist:
    96 86
     tests-debian-9:
    
    97 87
       image: buildstream/testsuite-debian:9-master-114-4cab18e3
    
    98 88
       <<: *linux-tests
    
    99
    -  <<: *tests-condition
    
    100 89
     
    
    101 90
     tests-fedora-27:
    
    102 91
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    103 92
       <<: *linux-tests
    
    104
    -  <<: *tests-condition
    
    105 93
     
    
    106 94
     tests-fedora-28:
    
    107 95
       image: buildstream/testsuite-fedora:28-master-114-4cab18e3
    
    108 96
       <<: *linux-tests
    
    109
    -  <<: *tests-condition
    
    110 97
     
    
    111 98
     tests-ubuntu-18.04:
    
    112 99
       image: buildstream/testsuite-ubuntu:18.04-master-114-4cab18e3
    
    113 100
       <<: *linux-tests
    
    114
    -  <<: *tests-condition
    
    115 101
     
    
    116 102
     tests-unix:
    
    117
    -  <<: *tests-condition
    
    118
    -
    
    119 103
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    
    120 104
       # can get rid of ostree - this is not possible with debian-8
    
    121 105
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    ... ... @@ -154,15 +138,6 @@ tests-unix:
    154 138
     # Note: We still do not enforce a consistent installation of python3-sphinx,
    
    155 139
     #       as it will significantly grow the backing image.
    
    156 140
     docs:
    
    157
    -
    
    158
    -  # Here we build the docs for every pre-merge CI, but avoid
    
    159
    -  # the job on post-merge to stable branches, because we only
    
    160
    -  # ever publish them from master
    
    161
    -  only:
    
    162
    -  - branches
    
    163
    -  except:
    
    164
    -  - /bst-1\..*/
    
    165
    -
    
    166 141
       stage: test
    
    167 142
       script:
    
    168 143
       - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
    
    ... ... @@ -178,19 +153,11 @@ docs:
    178 153
         - public/
    
    179 154
     
    
    180 155
     
    
    181
    -#####################################################
    
    182
    -#                    Post stage                     #
    
    183
    -#####################################################
    
    184
    -
    
    185
    -# Check code quality with codeclimate
    
    186
    -# This needs some refactoring; we probably just want to provide the codeclimate.json directly
    
    187
    -# as an output of radon, with some conversion
    
    156
    +# Check code quality with gitlab's built-in feature.
    
    188 157
     #
    
    189
    -codequality:
    
    190
    -  <<: *tests-condition
    
    191
    -
    
    158
    +code_quality:
    
    192 159
       image: docker:stable
    
    193
    -  stage: post
    
    160
    +  stage: test
    
    194 161
       variables:
    
    195 162
         DOCKER_DRIVER: overlay2
    
    196 163
       allow_failure: true
    
    ... ... @@ -204,11 +171,13 @@ codequality:
    204 171
             --volume /var/run/docker.sock:/var/run/docker.sock
    
    205 172
             "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
    
    206 173
       artifacts:
    
    207
    -    paths: [codeclimate.json]
    
    174
    +    paths: [gl-code-quality-report.json]
    
    208 175
     
    
    209
    -analysis:
    
    210
    -  <<: *tests-condition
    
    176
    +#####################################################
    
    177
    +#                    Post stage                     #
    
    178
    +#####################################################
    
    211 179
     
    
    180
    +analysis:
    
    212 181
       stage: post
    
    213 182
       script:
    
    214 183
       - |
    
    ... ... @@ -237,8 +206,6 @@ analysis:
    237 206
     # Collate coverage reports
    
    238 207
     #
    
    239 208
     coverage:
    
    240
    -  <<: *tests-condition
    
    241
    -
    
    242 209
       stage: post
    
    243 210
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    
    244 211
       script:
    

  • HACKING.rst
    ... ... @@ -390,6 +390,9 @@ Each *command* is a dictionary, the members of which are listed here:
    390 390
     
    
    391 391
     * ``command``: The command to run, without the leading ``bst``
    
    392 392
     
    
    393
    +* ``shell``: Specifying ``True`` indicates that ``command`` should be run as
    
    394
    +  a shell command from the project directory, instead of a bst command (optional)
    
    395
    +
    
    393 396
     When adding a new ``.run`` file, one should normally also commit the new
    
    394 397
     resulting generated ``.html`` file(s) into the ``doc/source/sessions-stored/``
    
    395 398
     directory at the same time, this ensures that other developers do not need to
    

  • README.rst
    1 1
     About
    
    2 2
     -----
    
    3
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/pipeline.svg
    
    4
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    5
    +
    
    6
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
    
    7
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    3 8
     
    
    4 9
     
    
    5 10
     What is BuildStream?
    
    ... ... @@ -69,10 +74,10 @@ BuildStream operates on a set of YAML files (.bst files), as follows:
    69 74
     
    
    70 75
     How can I get started?
    
    71 76
     ======================
    
    72
    -To start using BuildStream, first,
    
    73
    -`install <https://buildstream.gitlab.io/buildstream/main_install.html>`_
    
    74
    -BuildStream onto your machine and then follow our
    
    75
    -`tutorial <https://buildstream.gitlab.io/buildstream/using_tutorial.html>`_.
    
    77
    +To get started, first `install BuildStream by following the installation guide
    
    78
    +<https://buildstream.gitlab.io/buildstream/main_install.html>`_
    
    79
    +and then follow our tutorial in the
    
    80
    +`user guide <https://buildstream.gitlab.io/buildstream/main_using.html>`_.
    
    76 81
     
    
    77 82
     We also recommend exploring some existing BuildStream projects:
    
    78 83
     
    
    ... ... @@ -81,4 +86,3 @@ We also recommend exploring some existing BuildStream projects:
    81 86
     * https://gitlab.com/baserock/definitions
    
    82 87
     
    
    83 88
     If you have any questions please ask on our `#buildstream <irc://irc.gnome.org/buildstream>`_ channel in `irc.gnome.org <irc://irc.gnome.org>`_
    84
    -

  • buildstream/_project.py
    ... ... @@ -398,6 +398,11 @@ class Project():
    398 398
                     "Project requested format version {}, but BuildStream {}.{} only supports up until format version {}"
    
    399 399
                     .format(format_version, major, minor, BST_FORMAT_VERSION))
    
    400 400
     
    
    401
    +        # Check if project.conf contains name key field
    
    402
    +        if self._project_conf.get('name') is None:
    
    403
    +            raise LoadError(LoadErrorReason.INVALID_DATA,
    
    404
    +                            "{}: project.conf does not contain expected key '{}'".format(projectfile, 'name'))
    
    405
    +
    
    401 406
             # The project name, element path and option declarations
    
    402 407
             # are constant and cannot be overridden by option conditional statements
    
    403 408
             self.name = _yaml.node_get(pre_config_node, str, 'name')
    

  • buildstream/buildelement.py
    ... ... @@ -233,14 +233,13 @@ class BuildElement(Element):
    233 233
             return commands
    
    234 234
     
    
    235 235
         def __run_command(self, sandbox, cmd, cmd_name):
    
    236
    -        with self.timed_activity("Running {}".format(cmd_name)):
    
    237
    -            self.status("Running {}".format(cmd_name), detail=cmd)
    
    238
    -
    
    239
    -            # Note the -e switch to 'sh' means to exit with an error
    
    240
    -            # if any untested command fails.
    
    241
    -            #
    
    242
    -            exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    
    243
    -                                   SandboxFlags.ROOT_READ_ONLY)
    
    244
    -            if exitcode != 0:
    
    245
    -                raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    
    246
    -                                   collect=self.get_variable('install-root'))
    236
    +        self.status("Running {}".format(cmd_name), detail=cmd)
    
    237
    +
    
    238
    +        # Note the -e switch to 'sh' means to exit with an error
    
    239
    +        # if any untested command fails.
    
    240
    +        #
    
    241
    +        exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    
    242
    +                               SandboxFlags.ROOT_READ_ONLY)
    
    243
    +        if exitcode != 0:
    
    244
    +            raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    
    245
    +                               collect=self.get_variable('install-root'))

  • buildstream/element.py
    ... ... @@ -1101,9 +1101,12 @@ class Element(Plugin):
    1101 1101
                 # until the full cache query below.
    
    1102 1102
                 if (not self.__assemble_scheduled and not self.__assemble_done and
    
    1103 1103
                         not self.__cached_success(keystrength=_KeyStrength.WEAK) and
    
    1104
    -                    not self._pull_pending() and self._is_required()):
    
    1105
    -                self._schedule_assemble()
    
    1106
    -                return
    
    1104
    +                    not self._pull_pending()):
    
    1105
    +                # For uncached workspaced elements, assemble is required
    
    1106
    +                # even if we only need the cache key
    
    1107
    +                if self._is_required() or self._get_workspace():
    
    1108
    +                    self._schedule_assemble()
    
    1109
    +                    return
    
    1107 1110
     
    
    1108 1111
             if self.__strict_cache_key is None:
    
    1109 1112
                 dependencies = [
    
    ... ... @@ -1126,13 +1129,17 @@ class Element(Plugin):
    1126 1129
                     self.__weak_cached = self.__artifacts.contains(self, self.__weak_cache_key)
    
    1127 1130
     
    
    1128 1131
             if (not self.__assemble_scheduled and not self.__assemble_done and
    
    1129
    -                not self._cached_success() and not self._pull_pending() and self._is_required()):
    
    1132
    +                not self._cached_success() and not self._pull_pending()):
    
    1130 1133
                 # Workspaced sources are considered unstable if a build is pending
    
    1131 1134
                 # as the build will modify the contents of the workspace.
    
    1132 1135
                 # Determine as early as possible if a build is pending to discard
    
    1133 1136
                 # unstable cache keys.
    
    1134
    -            self._schedule_assemble()
    
    1135
    -            return
    
    1137
    +
    
    1138
    +            # For uncached workspaced elements, assemble is required
    
    1139
    +            # even if we only need the cache key
    
    1140
    +            if self._is_required() or self._get_workspace():
    
    1141
    +                self._schedule_assemble()
    
    1142
    +                return
    
    1136 1143
     
    
    1137 1144
             if self.__cache_key is None:
    
    1138 1145
                 # Calculate strong cache key
    
    ... ... @@ -1430,7 +1437,6 @@ class Element(Plugin):
    1430 1437
         # in a subprocess.
    
    1431 1438
         #
    
    1432 1439
         def _schedule_assemble(self):
    
    1433
    -        assert self._is_required()
    
    1434 1440
             assert not self.__assemble_scheduled
    
    1435 1441
             self.__assemble_scheduled = True
    
    1436 1442
     
    
    ... ... @@ -1438,6 +1444,8 @@ class Element(Plugin):
    1438 1444
             for dep in self.dependencies(Scope.BUILD, recurse=False):
    
    1439 1445
                 dep._set_required()
    
    1440 1446
     
    
    1447
    +        self._set_required()
    
    1448
    +
    
    1441 1449
             # Invalidate workspace key as the build modifies the workspace directory
    
    1442 1450
             workspace = self._get_workspace()
    
    1443 1451
             if workspace:
    
    ... ... @@ -1661,6 +1669,10 @@ class Element(Plugin):
    1661 1669
         #   (bool): Whether a pull operation is pending
    
    1662 1670
         #
    
    1663 1671
         def _pull_pending(self):
    
    1672
    +        if self._get_workspace():
    
    1673
    +            # Workspace builds are never pushed to artifact servers
    
    1674
    +            return False
    
    1675
    +
    
    1664 1676
             if self.__strong_cached:
    
    1665 1677
                 # Artifact already in local cache
    
    1666 1678
                 return False
    

  • doc/bst2html.py
    ... ... @@ -29,7 +29,7 @@ import sys
    29 29
     import re
    
    30 30
     import shlex
    
    31 31
     import subprocess
    
    32
    -from collections import Mapping
    
    32
    +from collections.abc import Mapping
    
    33 33
     from contextlib import contextmanager
    
    34 34
     from tempfile import TemporaryDirectory
    
    35 35
     
    

  • doc/source/developing/workspaces.rst
    ... ... @@ -4,9 +4,9 @@
    4 4
     
    
    5 5
     Workspaces
    
    6 6
     ==========
    
    7
    -
    
    8 7
     In this section we will cover the use of BuildStream's workspaces feature when devloping on a 
    
    9 8
     BuildStream project.
    
    9
    +
    
    10 10
     .. note::
    
    11 11
     
    
    12 12
        This example is distributed with BuildStream
    
    ... ... @@ -25,7 +25,6 @@ make changes to the source code of Buildstream elements by making use of
    25 25
     BuildStream's workspace command.
    
    26 26
     
    
    27 27
     
    
    28
    -
    
    29 28
     Opening a workspace
    
    30 29
     -------------------
    
    31 30
     First we need to open a workspace, we can do this by running
    
    ... ... @@ -37,12 +36,12 @@ This command has created the workspace_hello directory in which you can see
    37 36
     the source for the hello.bst element, i.e. hello.c and the corresponding
    
    38 37
     makefile.
    
    39 38
     
    
    40
    -
    
    41 39
     You can view existing workspaces using
    
    42 40
     
    
    43 41
     .. raw:: html
    
    44 42
        :file: ../sessions/developing-workspace-list.html
    
    45 43
     
    
    44
    +
    
    46 45
     Making code changes
    
    47 46
     -------------------
    
    48 47
     Let's say we want to alter the message printed when the hello command is run.
    
    ... ... @@ -66,6 +65,7 @@ This gives us the new message we changed in hello.c.
    66 65
     From this point we have several options. If the source is under version control
    
    67 66
     we can commit our changes and push them to the remote repository.
    
    68 67
     
    
    68
    +
    
    69 69
     Closing your workspace
    
    70 70
     ----------------------
    
    71 71
     If we want to close the workspace and come back to our changes later, we can
    

  • doc/source/install_linux_distro.rst
    1 1
     
    
    2
    -.. _install:
    
    3
    -
    
    4
    -Installing BuildStream on a Linux distro
    
    5
    -========================================
    
    6
    -BuildStream requires the following base system requirements:
    
    7
    -
    
    8
    -* python3 >= 3.5
    
    9
    -* bubblewrap >= 0.1.2
    
    10
    -* fuse2
    
    11
    -
    
    12
    -BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    
    13
    -Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    
    14
    -requirements of specific plugins.
    
    15
    -
    
    16
    -The default plugins with extra host dependencies are:
    
    17
    -
    
    18
    -* bzr
    
    19
    -* deb
    
    20
    -* git
    
    21
    -* ostree
    
    22
    -* patch
    
    23
    -* tar
    
    24
    -
    
    25
    -If you intend to push built artifacts to a remote artifact server,
    
    26
    -which requires special permissions, you will also need:
    
    27
    -
    
    28
    -* ssh
    
    29
    -
    
    30
    -
    
    31
    -Installing from source (recommended)
    
    32
    -------------------------------------
    
    33
    -Until BuildStream is available in your distro, you will need to install
    
    34
    -it yourself from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    
    35
    -using python's ``pip`` package manager.
    
    36
    -
    
    37
    -For the purpose of installing BuildStream while there are no distro packages,
    
    38
    -you will additionally need:
    
    39
    -
    
    40
    -* pip for python3 (only required for setup)
    
    41
    -* Python 3 development libraries and headers
    
    42
    -* git (to checkout buildstream)
    
    43
    -
    
    44
    -
    
    45
    -Installing dependencies
    
    46
    -~~~~~~~~~~~~~~~~~~~~~~~
    
    47
    -
    
    48
    -
    
    49
    -Arch Linux
    
    50
    -++++++++++
    
    51
    -Install the dependencies with::
    
    52
    -
    
    53
    -  sudo pacman -S \
    
    54
    -      python fuse2 bubblewrap \
    
    55
    -      python-pip git
    
    56
    -
    
    57
    -For the default plugins::
    
    58
    -
    
    59
    -  sudo pacman -S \
    
    60
    -      bzr git lzip ostree patch python-gobject
    
    61
    -
    
    62
    -
    
    63
    -The package *python-arpy* is required by the deb source plugin. This is not
    
    64
    -obtainable via `pacman`, you must get *python-arpy* from AUR:
    
    65
    -https://aur.archlinux.org/packages/python-arpy/
    
    66
    -
    
    67
    -To install::
    
    68
    -
    
    69
    -  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    
    70
    -  tar -xvf python-arpy.tar.gz
    
    71
    -  cd python-arpy
    
    72
    -  makepkg -si
    
    73
    -
    
    74
    -Debian
    
    75
    -++++++
    
    76
    -Install the dependencies with::
    
    77
    -
    
    78
    -  sudo apt-get install \
    
    79
    -      python3 fuse bubblewrap \
    
    80
    -      python3-pip python3-dev git
    
    81
    -
    
    82
    -For the default plugins:
    
    83
    -
    
    84
    -Stretch
    
    85
    -^^^^^^^
    
    86
    -With stretch, you first need to ensure that you have the backports repository
    
    87
    -setup as described `here <https://backports.debian.org/Instructions/>`_
    
    88
    -
    
    89
    -By adding the following line to your sources.list::
    
    90
    -
    
    91
    -  deb http://deb.debian.org/debian stretch-backports main
    
    92
    -
    
    93
    -And then running::
    
    94
    -
    
    95
    -  sudo apt update
    
    96
    -
    
    97
    -At this point you should be able to get the system requirements for the default plugins with::
    
    98
    -
    
    99
    -  sudo apt install \
    
    100
    -      bzr git lzip patch python3-arpy python3-gi
    
    101
    -  sudo apt install -t stretch-backports \
    
    102
    -      gir1.2-ostree-1.0 ostree
    
    103
    -
    
    104
    -Buster or Sid
    
    105
    -^^^^^^^^^^^^^
    
    106
    -For debian unstable or testing, only the following line should be enough
    
    107
    -to get the system requirements for the default plugins installed::
    
    108
    -
    
    109
    -  sudo apt-get install \
    
    110
    -      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    
    111
    -
    
    112
    -
    
    113
    -Fedora
    
    114
    -++++++
    
    115
    -For recent fedora systems, the following line should get you the system
    
    116
    -requirements you need::
    
    117
    -
    
    118
    -  dnf install -y \
    
    119
    -      python3 fuse bubblewrap \
    
    120
    -      python3-pip python3-devel git
    
    121
    -
    
    122
    -For the default plugins::
    
    123
    -
    
    124
    -  dnf install -y \
    
    125
    -      bzr git lzip patch ostree python3-gobject
    
    126
    -  pip3 install --user arpy
    
    127
    -
    
    128
    -
    
    129
    -Ubuntu
    
    130
    -++++++
    
    131
    -
    
    132
    -Ubuntu 18.04 LTS or later
    
    133
    -^^^^^^^^^^^^^^^^^^^^^^^^^
    
    134
    -Install the dependencies with::
    
    135
    -
    
    136
    -  sudo apt install \
    
    137
    -      python3 fuse bubblewrap \
    
    138
    -      python3-pip python3-dev git
    
    139
    -
    
    140
    -For the default plugins::
    
    141
    -
    
    142
    -  sudo apt install \
    
    143
    -      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    
    144
    -
    
    145
    -Ubuntu 16.04 LTS
    
    146
    -^^^^^^^^^^^^^^^^
    
    147
    -On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    
    148
    -or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    
    149
    -You will need to install them in whichever way you see fit. Refer the the upstream documentation
    
    150
    -for advice on this.
    
    151
    -
    
    152
    -
    
    153
    -Installing
    
    154
    -~~~~~~~~~~
    
    155
    -Once you have the base system dependencies, you can install the BuildStream
    
    156
    -python package as a regular user.
    
    157
    -
    
    158
    -Via PyPI (recommended)
    
    159
    -++++++++++++++++++++++
    
    160
    -::
    
    161
    -
    
    162
    -  pip3 install --user BuildStream
    
    163
    -
    
    164
    -This will install latest stable version of BuildStream and its pure python
    
    165
    -dependencies into your user's homedir in ``~/.local``.
    
    166
    -
    
    167
    -Keep following the instructions below to ensure that the ``bst``
    
    168
    -command is in your ``PATH`` and to enable bash completions for it.
    
    169
    -
    
    170
    -.. note::
    
    171
    -
    
    172
    -  If you want a specific version of BuildStream, you can install it using
    
    173
    -  ``pip install --user BuildStream==<version-number>``
    
    174
    -
    
    175
    -Via Git checkout
    
    176
    -++++++++++++++++
    
    177
    -::
    
    178
    -
    
    179
    -  git clone https://gitlab.com/BuildStream/buildstream.git
    
    180
    -  cd buildstream
    
    181
    -  pip3 install --user -e .
    
    182
    -
    
    183
    -This will install buildstream's pure python dependencies into
    
    184
    -your user's homedir in ``~/.local`` and will run BuildStream directly
    
    185
    -from the git checkout directory.
    
    186
    -
    
    187
    -Keep following the instructions below to ensure that the ``bst``
    
    188
    -command is in your ``PATH`` and to enable bash completions for it.
    
    189
    -
    
    190
    -.. note::
    
    191
    -
    
    192
    -   We recommend the ``-e`` option because you can upgrade your
    
    193
    -   installation by simply updating the checked out git repository.
    
    194
    -
    
    195
    -   If you want a full installation that is not linked to your
    
    196
    -   git checkout, just omit the ``-e`` option from the above commands.
    
    197
    -
    
    198
    -
    
    199
    -Adjust PATH
    
    200
    -~~~~~~~~~~~
    
    201
    -Since BuildStream is now installed under your local user's install directories,
    
    202
    -you need to ensure that ``PATH`` is adjusted.
    
    203
    -
    
    204
    -A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    
    205
    -
    
    206
    -  export PATH="${PATH}:${HOME}/.local/bin"
    
    207
    -
    
    208
    -.. note::
    
    209
    -
    
    210
    -   You will have to restart your terminal in order for these changes to take effect.
    
    211
    -
    
    212
    -
    
    213
    -Bash completions
    
    214
    -~~~~~~~~~~~~~~~~
    
    215
    -Bash completions are supported by sourcing the ``buildstream/data/bst``
    
    216
    -script found in the BuildStream repository. On many systems this script
    
    217
    -can be installed into a completions directory but when installing BuildStream
    
    218
    -without a package manager this is not an option.
    
    219
    -
    
    220
    -To enable completions for an installation of BuildStream you
    
    221
    -installed yourself from git, just append the script verbatim
    
    222
    -to your ``~/.bash_completion``:
    
    223
    -
    
    224
    -.. literalinclude:: ../../buildstream/data/bst
    
    225
    -   :language: yaml
    
    226
    -
    
    227
    -
    
    228
    -Upgrading BuildStream
    
    229
    -~~~~~~~~~~~~~~~~~~~~~
    
    230
    -
    
    231
    -Via PyPI
    
    232
    -++++++++
    
    233
    -
    
    234
    -If you installed BuildStream from PyPI, you can update it like so::
    
    235
    -
    
    236
    -  pip install --user --upgrade BuildStream
    
    237
    -
    
    238
    -Via Git checkout
    
    239
    -++++++++++++++++
    
    240
    -
    
    241
    -If you installed BuildStream from a local git checkout using ``-e`` option, all
    
    242
    -you need to do to upgrade BuildStream is to update your local git checkout::
    
    243
    -
    
    244
    -  cd /path/to/buildstream
    
    245
    -  git pull --rebase
    
    246
    -
    
    247
    -If you did not specify the ``-e`` option at install time or the dependancies
    
    248
    -have changed, you will need to cleanly reinstall BuildStream::
    
    249
    -
    
    250
    -  pip3 uninstall buildstream
    
    251
    -  cd /path/to/buildstream
    
    252
    -  git pull --rebase
    
    253
    -  pip3 install --user .
    
    254 2
     
    
    3
    +.. _install_linux_distro:
    
    255 4
     
    
    256 5
     Installing from distro packages
    
    257
    --------------------------------
    
    6
    +===============================
    
    7
    +BuildStream is available on some linux distributions, here are
    
    8
    +some install instructions for the linux distributions which
    
    9
    +have packaged BuildStream.
    
    258 10
     
    
    259 11
     
    
    260 12
     Arch Linux
    
    261
    -~~~~~~~~~~
    
    13
    +----------
    
    262 14
     Packages for Arch exist in `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_packages>`_.
    
    263 15
     Two different package versions are available:
    
    264 16
     
    
    ... ... @@ -267,8 +19,7 @@ Two different package versions are available:
    267 19
     
    
    268 20
     
    
    269 21
     Fedora
    
    270
    -~~~~~~
    
    271
    -
    
    22
    +------
    
    272 23
     BuildStream is not yet in the official Fedora repositories, but you can
    
    273 24
     install it from a Copr::
    
    274 25
     
    

  • doc/source/install_source.rst
    1
    +
    
    2
    +
    
    3
    +Installing from source
    
    4
    +======================
    
    5
    +Until BuildStream is available in :ref:`your distro <install_linux_distro>`, you will
    
    6
    +need to install it yourself from source.
    
    7
    +
    
    8
    +
    
    9
    +Installing dependencies
    
    10
    +-----------------------
    
    11
    +Before installing BuildStream from source, it is necessary to first install
    
    12
    +the system dependencies. Below are some linux distribution specific instructions
    
    13
    +for installing these dependencies.
    
    14
    +
    
    15
    +BuildStream requires the following base system requirements:
    
    16
    +
    
    17
    +* python3 >= 3.5
    
    18
    +* bubblewrap >= 0.1.2
    
    19
    +* fuse2
    
    20
    +
    
    21
    +BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    
    22
    +Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    
    23
    +requirements of specific plugins.
    
    24
    +
    
    25
    +The default plugins with extra host dependencies are:
    
    26
    +
    
    27
    +* bzr
    
    28
    +* deb
    
    29
    +* git
    
    30
    +* ostree
    
    31
    +* patch
    
    32
    +* tar
    
    33
    +
    
    34
    +If you intend to push built artifacts to a remote artifact server,
    
    35
    +which requires special permissions, you will also need:
    
    36
    +
    
    37
    +* ssh
    
    38
    +
    
    39
    +
    
    40
    +Arch Linux
    
    41
    +~~~~~~~~~~
    
    42
    +Install the dependencies with::
    
    43
    +
    
    44
    +  sudo pacman -S \
    
    45
    +      python fuse2 bubblewrap \
    
    46
    +      python-pip
    
    47
    +
    
    48
    +For the default plugins::
    
    49
    +
    
    50
    +  sudo pacman -S \
    
    51
    +      bzr git lzip ostree patch python-gobject
    
    52
    +
    
    53
    +
    
    54
    +The package *python-arpy* is required by the deb source plugin. This is not
    
    55
    +obtainable via `pacman`, you must get *python-arpy* from AUR:
    
    56
    +https://aur.archlinux.org/packages/python-arpy/
    
    57
    +
    
    58
    +To install::
    
    59
    +
    
    60
    +  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    
    61
    +  tar -xvf python-arpy.tar.gz
    
    62
    +  cd python-arpy
    
    63
    +  makepkg -si
    
    64
    +
    
    65
    +
    
    66
    +Debian
    
    67
    +~~~~~~
    
    68
    +Install the dependencies with::
    
    69
    +
    
    70
    +  sudo apt-get install \
    
    71
    +      python3 fuse bubblewrap \
    
    72
    +      python3-pip python3-dev
    
    73
    +
    
    74
    +For the default plugins:
    
    75
    +
    
    76
    +
    
    77
    +Stretch
    
    78
    ++++++++
    
    79
    +With stretch, you first need to ensure that you have the backports repository
    
    80
    +setup as described `here <https://backports.debian.org/Instructions/>`_
    
    81
    +
    
    82
    +By adding the following line to your sources.list::
    
    83
    +
    
    84
    +  deb http://deb.debian.org/debian stretch-backports main
    
    85
    +
    
    86
    +And then running::
    
    87
    +
    
    88
    +  sudo apt update
    
    89
    +
    
    90
    +At this point you should be able to get the system requirements for the default plugins with::
    
    91
    +
    
    92
    +  sudo apt install \
    
    93
    +      bzr git lzip patch python3-arpy python3-gi
    
    94
    +  sudo apt install -t stretch-backports \
    
    95
    +      gir1.2-ostree-1.0 ostree
    
    96
    +
    
    97
    +
    
    98
    +Buster or Sid
    
    99
    ++++++++++++++
    
    100
    +For debian unstable or testing, only the following line should be enough
    
    101
    +to get the system requirements for the default plugins installed::
    
    102
    +
    
    103
    +  sudo apt-get install \
    
    104
    +      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    
    105
    +
    
    106
    +
    
    107
    +Fedora
    
    108
    +~~~~~~
    
    109
    +For recent fedora systems, the following line should get you the system
    
    110
    +requirements you need::
    
    111
    +
    
    112
    +  dnf install -y \
    
    113
    +      python3 fuse bubblewrap \
    
    114
    +      python3-pip python3-devel
    
    115
    +
    
    116
    +For the default plugins::
    
    117
    +
    
    118
    +  dnf install -y \
    
    119
    +      bzr git lzip patch ostree python3-gobject
    
    120
    +  pip3 install --user arpy
    
    121
    +
    
    122
    +
    
    123
    +Ubuntu
    
    124
    +~~~~~~
    
    125
    +
    
    126
    +
    
    127
    +Ubuntu 18.04 LTS or later
    
    128
    ++++++++++++++++++++++++++
    
    129
    +Install the dependencies with::
    
    130
    +
    
    131
    +  sudo apt install \
    
    132
    +      python3 fuse bubblewrap \
    
    133
    +      python3-pip python3-dev
    
    134
    +
    
    135
    +For the default plugins::
    
    136
    +
    
    137
    +  sudo apt install \
    
    138
    +      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    
    139
    +
    
    140
    +
    
    141
    +Ubuntu 16.04 LTS
    
    142
    +++++++++++++++++
    
    143
    +On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    
    144
    +or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    
    145
    +You will need to install them in whichever way you see fit. Refer the the upstream documentation
    
    146
    +for advice on this.
    
    147
    +
    
    148
    +
    
    149
    +Installing
    
    150
    +----------
    
    151
    +Once you have the base system dependencies, you can install the BuildStream
    
    152
    +python package as a regular user.
    
    153
    +
    
    154
    +
    
    155
    +Installing from PyPI (recommended)
    
    156
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    157
    +Since we only ever publish :ref:`release versions <install_semantic_versioning>` on
    
    158
    +PyPI, it is currently recommended to use this installation path. This will
    
    159
    +ensure that you always have the latest recommended version of BuildStream that
    
    160
    +we recommend.
    
    161
    +
    
    162
    +To install from PyPI, you will additionally require:
    
    163
    +
    
    164
    +* pip for python3 (only required for setup)
    
    165
    +* Python 3 development libraries and headers
    
    166
    +
    
    167
    +Simply run the following command::
    
    168
    +
    
    169
    +  pip3 install --user BuildStream
    
    170
    +
    
    171
    +This will install latest stable version of BuildStream and its pure python
    
    172
    +dependencies into your user's homedir in ``~/.local``.
    
    173
    +
    
    174
    +Keep following the instructions below to ensure that the ``bst``
    
    175
    +command is in your ``PATH`` and to enable bash completions for it.
    
    176
    +
    
    177
    +.. note::
    
    178
    +
    
    179
    +  If you want a specific version of BuildStream, you can install it using
    
    180
    +  ``pip install --user BuildStream==<version-number>``
    
    181
    +
    
    182
    +
    
    183
    +Upgrading from PyPI
    
    184
    ++++++++++++++++++++
    
    185
    +Once you have already installed BuildStream from PyPI, you can later update
    
    186
    +to the latest recommended version like so::
    
    187
    +
    
    188
    +  pip install --user --upgrade BuildStream
    
    189
    +
    
    190
    +
    
    191
    +.. _install_git_checkout:
    
    192
    +
    
    193
    +Installing from a git checkout
    
    194
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    195
    +To install directly from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    
    196
    +using python's ``pip`` package manager, you will additionally require:
    
    197
    +
    
    198
    +* pip for python3 (only required for setup)
    
    199
    +* Python 3 development libraries and headers
    
    200
    +* git (to checkout BuildStream)
    
    201
    +
    
    202
    +Before installing, please check the existing tags in the git repository
    
    203
    +and determine which version you want to install, and whether you want
    
    204
    +to install an official release version (recommended), or a development snapshot
    
    205
    +to help us out testing the bleeding edge of development. Follow the
    
    206
    +:ref:`semantic versioning guide <install_semantic_versioning>` to determine
    
    207
    +which tag you intend to install.
    
    208
    +
    
    209
    +Run the following commands::
    
    210
    +
    
    211
    +  git clone https://gitlab.com/BuildStream/buildstream.git
    
    212
    +  cd buildstream
    
    213
    +  git checkout <desired release tag>
    
    214
    +  pip3 install --user -e .
    
    215
    +
    
    216
    +This will install buildstream's pure python dependencies into
    
    217
    +your user's homedir in ``~/.local`` and will run BuildStream directly
    
    218
    +from the git checkout directory.
    
    219
    +
    
    220
    +Keep following the instructions below to ensure that the ``bst``
    
    221
    +command is in your ``PATH`` and to enable bash completions for it.
    
    222
    +
    
    223
    +.. note::
    
    224
    +
    
    225
    +   We recommend the ``-e`` option because you can upgrade your
    
    226
    +   installation by simply updating the checked out git repository.
    
    227
    +
    
    228
    +   If you want a full installation that is not linked to your
    
    229
    +   git checkout, just omit the ``-e`` option from the above commands.
    
    230
    +
    
    231
    +
    
    232
    +Upgrading from a git checkout
    
    233
    ++++++++++++++++++++++++++++++
    
    234
    +If you installed BuildStream from a local git checkout using ``-e`` option, all
    
    235
    +you need to do to upgrade BuildStream is to update your local git checkout::
    
    236
    +
    
    237
    +  cd /path/to/buildstream
    
    238
    +  git pull --rebase
    
    239
    +
    
    240
    +If you did not specify the ``-e`` option at install time or the dependancies
    
    241
    +have changed, you will need to cleanly reinstall BuildStream::
    
    242
    +
    
    243
    +  pip3 uninstall buildstream
    
    244
    +  cd /path/to/buildstream
    
    245
    +  git pull --rebase
    
    246
    +  pip3 install --user .
    
    247
    +
    
    248
    +.. note::
    
    249
    +
    
    250
    +   If BuildStream has added any dependencies since the last upgrade,
    
    251
    +   you will need to uninstall and reinstall to ensure those dependencies
    
    252
    +   are met, regardless of whether you have used the ``-e`` option at
    
    253
    +   install time.
    
    254
    +
    
    255
    +
    
    256
    +Post install setup
    
    257
    +------------------
    
    258
    +After having installed from source using any of the above methods, some
    
    259
    +setup will be required to use BuildStream.
    
    260
    +
    
    261
    +
    
    262
    +Adjust PATH
    
    263
    +~~~~~~~~~~~
    
    264
    +Since BuildStream is now installed under your local user's install directories,
    
    265
    +you need to ensure that ``PATH`` is adjusted.
    
    266
    +
    
    267
    +A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    
    268
    +
    
    269
    +  export PATH="${PATH}:${HOME}/.local/bin"
    
    270
    +
    
    271
    +.. note::
    
    272
    +
    
    273
    +   You will have to restart your terminal in order for these changes to take effect.
    
    274
    +
    
    275
    +
    
    276
    +Bash completions
    
    277
    +~~~~~~~~~~~~~~~~
    
    278
    +Bash completions are supported by sourcing the ``buildstream/data/bst``
    
    279
    +script found in the BuildStream repository. On many systems this script
    
    280
    +can be installed into a completions directory but when installing BuildStream
    
    281
    +without a package manager this is not an option.
    
    282
    +
    
    283
    +To enable completions for an installation of BuildStream you
    
    284
    +installed yourself from git, just append the script verbatim
    
    285
    +to your ``~/.bash_completion``:
    
    286
    +
    
    287
    +.. literalinclude:: ../../buildstream/data/bst
    
    288
    +   :language: yaml

  • doc/source/install_versions.rst
    1
    +
    
    2
    +
    
    3
    +.. _install_semantic_versioning:
    
    4
    +
    
    5
    +Semantic Versioning
    
    6
    +===================
    
    7
    +BuildStream follows the Semantic Versioning Convention `(SemVer) <https://semver.org/>`_,
    
    8
    +and uses even minor point numbers to denote releases intended for users while
    
    9
    +odd minor point numbers represent development snapshops.
    
    10
    +
    
    11
    +For example, for a given version number ``X.Y.Z``
    
    12
    + * The ``X.<even number>.*`` versions are releases intended for users.
    
    13
    + * The ``X.<odd number>.*`` versions are development spanshots intended for testing.
    
    14
    +
    
    15
    +If you are :ref:`installing from git <install_git_checkout>`, please look for the latest
    
    16
    +tag in the latest release branch to ensure you're getting the latest release.
    
    17
    +
    
    18
    +Current release branches:
    
    19
    + * `bst-1.2 (latest) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.2>`_
    
    20
    + * `bst-1.0 (deprecated) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.0>`_

  • doc/source/main_install.rst
    1
    +
    
    2
    +
    
    3
    +.. _install:
    
    4
    +
    
    1 5
     Install
    
    2 6
     =======
    
    3
    -This section covers how to install BuildStream onto your machine, how to run
    
    4
    -BuildStream inside a docker image and also how to configure an artifact server.
    
    7
    +This section provides instructions for installing BuildStream and its
    
    8
    +companion artifact server on various platforms, along with any installation
    
    9
    +related materials.
    
    5 10
     
    
    6 11
     .. note::
    
    7 12
     
    
    8
    -   BuildStream is not currently supported natively on macOS and Windows. Windows
    
    9
    -   and macOS users should refer to :ref:`docker`.
    
    13
    +   BuildStream is currently only supported natively on Linux. Users of Unix-like
    
    14
    +   systems where Docker is available can still use BuildStream by following the
    
    15
    +   :ref:`Docker install guide <docker>`
    
    10 16
     
    
    11 17
     .. toctree::
    
    12
    -   :maxdepth: 2
    
    18
    +   :maxdepth: 1
    
    13 19
     
    
    20
    +   install_source
    
    14 21
        install_linux_distro
    
    15 22
        install_docker
    
    16 23
        install_artifacts
    
    24
    +   install_versions

  • doc/source/sessions/developing-build-after-changes.htmldoc/source/sessions-stored/developing-build-after-changes.html

  • doc/source/sessions/developing-close-workspace.htmldoc/source/sessions-stored/developing-close-workspace.html

  • doc/source/sessions/developing-discard-workspace.htmldoc/source/sessions-stored/developing-discard-workspace.html

  • doc/source/sessions/developing-reopen-workspace.htmldoc/source/sessions-stored/developing-reopen-workspace.html

  • doc/source/sessions/developing-reset-workspace.htmldoc/source/sessions-stored/developing-reset-workspace.html

  • doc/source/sessions/developing-shell-after-changes.htmldoc/source/sessions-stored/developing-shell-after-changes.html

  • doc/source/sessions/developing-workspace-list.htmldoc/source/sessions-stored/developing-workspace-list.html

  • doc/source/sessions/developing-workspace-open.htmldoc/source/sessions-stored/developing-workspace-open.html

  • setup.cfg
    ... ... @@ -11,7 +11,7 @@ parentdir_prefix = BuildStream-
    11 11
     test=pytest
    
    12 12
     
    
    13 13
     [tool:pytest]
    
    14
    -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc
    
    14
    +addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
    
    15 15
     norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
    
    16 16
     python_files = tests/*/*.py
    
    17 17
     pep8maxlinelength = 119
    

  • tests/frontend/workspace.py
    ... ... @@ -780,3 +780,73 @@ def test_inconsitent_pipeline_message(cli, tmpdir, datafiles, kind):
    780 780
             'build', element_name
    
    781 781
         ])
    
    782 782
         result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline-workspaced")
    
    783
    +
    
    784
    +
    
    785
    +@pytest.mark.datafiles(DATA_DIR)
    
    786
    +@pytest.mark.parametrize("strict", [("strict"), ("non-strict")])
    
    787
    +def test_cache_key_workspace_in_dependencies(cli, tmpdir, datafiles, strict):
    
    788
    +    checkout = os.path.join(str(tmpdir), 'checkout')
    
    789
    +    element_name, project, workspace = open_workspace(cli, os.path.join(str(tmpdir), 'repo-a'),
    
    790
    +                                                      datafiles, 'git', False)
    
    791
    +
    
    792
    +    element_path = os.path.join(project, 'elements')
    
    793
    +    back_dep_element_name = 'workspace-test-back-dep.bst'
    
    794
    +
    
    795
    +    # Write out our test target
    
    796
    +    element = {
    
    797
    +        'kind': 'compose',
    
    798
    +        'depends': [
    
    799
    +            {
    
    800
    +                'filename': element_name,
    
    801
    +                'type': 'build'
    
    802
    +            }
    
    803
    +        ]
    
    804
    +    }
    
    805
    +    _yaml.dump(element,
    
    806
    +               os.path.join(element_path,
    
    807
    +                            back_dep_element_name))
    
    808
    +
    
    809
    +    # Modify workspace
    
    810
    +    shutil.rmtree(os.path.join(workspace, 'usr', 'bin'))
    
    811
    +    os.makedirs(os.path.join(workspace, 'etc'))
    
    812
    +    with open(os.path.join(workspace, 'etc', 'pony.conf'), 'w') as f:
    
    813
    +        f.write("PONY='pink'")
    
    814
    +
    
    815
    +    # Configure strict mode
    
    816
    +    strict_mode = True
    
    817
    +    if strict != 'strict':
    
    818
    +        strict_mode = False
    
    819
    +    cli.configure({
    
    820
    +        'projects': {
    
    821
    +            'test': {
    
    822
    +                'strict': strict_mode
    
    823
    +            }
    
    824
    +        }
    
    825
    +    })
    
    826
    +
    
    827
    +    # Build artifact with dependency's modified workspace
    
    828
    +    assert cli.get_element_state(project, element_name) == 'buildable'
    
    829
    +    assert cli.get_element_key(project, element_name) == "{:?<64}".format('')
    
    830
    +    assert cli.get_element_state(project, back_dep_element_name) == 'waiting'
    
    831
    +    assert cli.get_element_key(project, back_dep_element_name) == "{:?<64}".format('')
    
    832
    +    result = cli.run(project=project, args=['build', back_dep_element_name])
    
    833
    +    result.assert_success()
    
    834
    +    assert cli.get_element_state(project, element_name) == 'cached'
    
    835
    +    assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
    
    836
    +    assert cli.get_element_state(project, back_dep_element_name) == 'cached'
    
    837
    +    assert cli.get_element_key(project, back_dep_element_name) != "{:?<64}".format('')
    
    838
    +    result = cli.run(project=project, args=['build', back_dep_element_name])
    
    839
    +    result.assert_success()
    
    840
    +
    
    841
    +    # Checkout the result
    
    842
    +    result = cli.run(project=project, args=[
    
    843
    +        'checkout', back_dep_element_name, checkout
    
    844
    +    ])
    
    845
    +    result.assert_success()
    
    846
    +
    
    847
    +    # Check that the pony.conf from the modified workspace exists
    
    848
    +    filename = os.path.join(checkout, 'etc', 'pony.conf')
    
    849
    +    assert os.path.exists(filename)
    
    850
    +
    
    851
    +    # Check that the original /usr/bin/hello is not in the checkout
    
    852
    +    assert not os.path.exists(os.path.join(checkout, 'usr', 'bin', 'hello'))



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]