[Notes] [Git][BuildStream/buildstream][Qinusty/600-recursive-variables] 43 commits: .gitlab-ci.yml: Verify that source tarball installs correctly



Title: GitLab

Qinusty pushed to branch Qinusty/600-recursive-variables at BuildStream / buildstream

Commits:

30 changed files:

Changes:

  • .gitignore
    ... ... @@ -25,7 +25,8 @@ __pycache__/
    25 25
     # Generated version file
    
    26 26
     buildstream/__version__.py
    
    27 27
     
    
    28
    -#Autogenerated doc
    
    28
    +# Autogenerated doc
    
    29
    +doc/source/sessions/
    
    29 30
     doc/source/elements/
    
    30 31
     doc/source/sources/
    
    31 32
     doc/source/modules.rst
    

  • .gitlab-ci.yml
    ... ... @@ -10,16 +10,6 @@ stages:
    10 10
       - test
    
    11 11
       - post
    
    12 12
     
    
    13
    -# Avoid running all the tests post merge on
    
    14
    -# master or on any release branch.
    
    15
    -#
    
    16
    -.tests-condition-template: &tests-condition
    
    17
    -  only:
    
    18
    -  - branches
    
    19
    -  except:
    
    20
    -  - master
    
    21
    -  - /bst-1\..*/
    
    22
    -
    
    23 13
     #####################################################
    
    24 14
     #                  Prepare stage                    #
    
    25 15
     #####################################################
    
    ... ... @@ -36,6 +26,11 @@ source_dist:
    36 26
       - tar -ztf dist/*
    
    37 27
       - tarball=$(cd dist && echo $(ls *))
    
    38 28
     
    
    29
    +  # Verify that the source distribution tarball can be installed correctly
    
    30
    +  #
    
    31
    +  - pip3 install dist/*.tar.gz
    
    32
    +  - bst --version
    
    33
    +
    
    39 34
       # unpack tarball as `dist/buildstream` directory
    
    40 35
       - |
    
    41 36
         cat > dist/unpack.sh << EOF
    
    ... ... @@ -91,26 +86,20 @@ source_dist:
    91 86
     tests-debian-9:
    
    92 87
       image: buildstream/testsuite-debian:9-master-114-4cab18e3
    
    93 88
       <<: *linux-tests
    
    94
    -  <<: *tests-condition
    
    95 89
     
    
    96 90
     tests-fedora-27:
    
    97 91
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    98 92
       <<: *linux-tests
    
    99
    -  <<: *tests-condition
    
    100 93
     
    
    101 94
     tests-fedora-28:
    
    102 95
       image: buildstream/testsuite-fedora:28-master-114-4cab18e3
    
    103 96
       <<: *linux-tests
    
    104
    -  <<: *tests-condition
    
    105 97
     
    
    106 98
     tests-ubuntu-18.04:
    
    107 99
       image: buildstream/testsuite-ubuntu:18.04-master-114-4cab18e3
    
    108 100
       <<: *linux-tests
    
    109
    -  <<: *tests-condition
    
    110 101
     
    
    111 102
     tests-unix:
    
    112
    -  <<: *tests-condition
    
    113
    -
    
    114 103
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    
    115 104
       # can get rid of ostree - this is not possible with debian-8
    
    116 105
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    ... ... @@ -149,15 +138,6 @@ tests-unix:
    149 138
     # Note: We still do not enforce a consistent installation of python3-sphinx,
    
    150 139
     #       as it will significantly grow the backing image.
    
    151 140
     docs:
    
    152
    -
    
    153
    -  # Here we build the docs for every pre-merge CI, but avoid
    
    154
    -  # the job on post-merge to stable branches, because we only
    
    155
    -  # ever publish them from master
    
    156
    -  only:
    
    157
    -  - branches
    
    158
    -  except:
    
    159
    -  - /bst-1\..*/
    
    160
    -
    
    161 141
       stage: test
    
    162 142
       script:
    
    163 143
       - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
    
    ... ... @@ -173,19 +153,11 @@ docs:
    173 153
         - public/
    
    174 154
     
    
    175 155
     
    
    176
    -#####################################################
    
    177
    -#                    Post stage                     #
    
    178
    -#####################################################
    
    179
    -
    
    180
    -# Check code quality with codeclimate
    
    181
    -# This needs some refactoring; we probably just want to provide the codeclimate.json directly
    
    182
    -# as an output of radon, with some conversion
    
    156
    +# Check code quality with gitlab's built-in feature.
    
    183 157
     #
    
    184
    -codequality:
    
    185
    -  <<: *tests-condition
    
    186
    -
    
    158
    +code_quality:
    
    187 159
       image: docker:stable
    
    188
    -  stage: post
    
    160
    +  stage: test
    
    189 161
       variables:
    
    190 162
         DOCKER_DRIVER: overlay2
    
    191 163
       allow_failure: true
    
    ... ... @@ -199,11 +171,13 @@ codequality:
    199 171
             --volume /var/run/docker.sock:/var/run/docker.sock
    
    200 172
             "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
    
    201 173
       artifacts:
    
    202
    -    paths: [codeclimate.json]
    
    174
    +    paths: [gl-code-quality-report.json]
    
    203 175
     
    
    204
    -analysis:
    
    205
    -  <<: *tests-condition
    
    176
    +#####################################################
    
    177
    +#                    Post stage                     #
    
    178
    +#####################################################
    
    206 179
     
    
    180
    +analysis:
    
    207 181
       stage: post
    
    208 182
       script:
    
    209 183
       - |
    
    ... ... @@ -232,8 +206,6 @@ analysis:
    232 206
     # Collate coverage reports
    
    233 207
     #
    
    234 208
     coverage:
    
    235
    -  <<: *tests-condition
    
    236
    -
    
    237 209
       stage: post
    
    238 210
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    
    239 211
       script:
    

  • HACKING.rst
    ... ... @@ -390,6 +390,9 @@ Each *command* is a dictionary, the members of which are listed here:
    390 390
     
    
    391 391
     * ``command``: The command to run, without the leading ``bst``
    
    392 392
     
    
    393
    +* ``shell``: Specifying ``True`` indicates that ``command`` should be run as
    
    394
    +  a shell command from the project directory, instead of a bst command (optional)
    
    395
    +
    
    393 396
     When adding a new ``.run`` file, one should normally also commit the new
    
    394 397
     resulting generated ``.html`` file(s) into the ``doc/source/sessions-stored/``
    
    395 398
     directory at the same time, this ensures that other developers do not need to
    
    ... ... @@ -465,12 +468,12 @@ If you want to run a specific test or a group of tests, you
    465 468
     can specify a prefix to match. E.g. if you want to run all of
    
    466 469
     the frontend tests you can do::
    
    467 470
     
    
    468
    -  ./setup.py test --addopts '-k tests/frontend/'
    
    471
    +  ./setup.py test --addopts 'tests/frontend/'
    
    469 472
     
    
    470 473
     Specific tests can be chosen by using the :: delimeter after the test module.
    
    471 474
     If you wanted to run the test_build_track test within frontend/buildtrack.py you could do::
    
    472 475
     
    
    473
    -  ./setup.py test --adopts '-k tests/frontend/buildtrack.py::test_build_track'
    
    476
    +  ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
    
    474 477
     
    
    475 478
     We also have a set of slow integration tests that are disabled by
    
    476 479
     default - you will notice most of them marked with SKIP in the pytest
    

  • README.rst
    1 1
     About
    
    2 2
     -----
    
    3
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/pipeline.svg
    
    4
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    5
    +
    
    6
    +.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
    
    7
    +   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    3 8
     
    
    4 9
     
    
    5 10
     What is BuildStream?
    
    ... ... @@ -69,10 +74,10 @@ BuildStream operates on a set of YAML files (.bst files), as follows:
    69 74
     
    
    70 75
     How can I get started?
    
    71 76
     ======================
    
    72
    -To start using BuildStream, first,
    
    73
    -`install <https://buildstream.gitlab.io/buildstream/main_install.html>`_
    
    74
    -BuildStream onto your machine and then follow our
    
    75
    -`tutorial <https://buildstream.gitlab.io/buildstream/using_tutorial.html>`_.
    
    77
    +To get started, first `install BuildStream by following the installation guide
    
    78
    +<https://buildstream.gitlab.io/buildstream/main_install.html>`_
    
    79
    +and then follow our tutorial in the
    
    80
    +`user guide <https://buildstream.gitlab.io/buildstream/main_using.html>`_.
    
    76 81
     
    
    77 82
     We also recommend exploring some existing BuildStream projects:
    
    78 83
     
    
    ... ... @@ -81,4 +86,3 @@ We also recommend exploring some existing BuildStream projects:
    81 86
     * https://gitlab.com/baserock/definitions
    
    82 87
     
    
    83 88
     If you have any questions please ask on our `#buildstream <irc://irc.gnome.org/buildstream>`_ channel in `irc.gnome.org <irc://irc.gnome.org>`_
    84
    -

  • buildstream/_exceptions.py
    ... ... @@ -217,6 +217,9 @@ class LoadErrorReason(Enum):
    217 217
         # A recursive include has been encountered.
    
    218 218
         RECURSIVE_INCLUDE = 21
    
    219 219
     
    
    220
    +    # A recursive variable has been encountered
    
    221
    +    RECURSIVE_VARIABLE = 22
    
    222
    +
    
    220 223
     
    
    221 224
     # LoadError
    
    222 225
     #
    

  • buildstream/_variables.py
    ... ... @@ -132,6 +132,12 @@ class Variables():
    132 132
                     value = _yaml.node_get(variables, str, key)
    
    133 133
     
    
    134 134
                     resolved_var, item_unmatched = self._subst(value, variables)
    
    135
    +                if _wrap_variable(key) in resolved_var:
    
    136
    +                    raise LoadError(LoadErrorReason.RECURSIVE_VARIABLE,
    
    137
    +                                    "{}: ".format(_yaml.node_get_provenance(variables, key)) +
    
    138
    +                                    ("Variable '{}' references itself recursively through" +
    
    139
    +                                     " another variable declariation.").format(key))
    
    140
    +
    
    135 141
                     resolved[key] = resolved_var
    
    136 142
                     unmatched += item_unmatched
    
    137 143
     
    
    ... ... @@ -162,14 +168,17 @@ class Variables():
    162 168
                                     "Failed to resolve one or more variable:\n{}".format(summary))
    
    163 169
     
    
    164 170
                 last_unmatched = unmatched
    
    165
    -
    
    166 171
             return resolved
    
    167 172
     
    
    168 173
         # Helper function to fetch information about the node referring to a variable
    
    169 174
         #
    
    170 175
         def _find_references(self, varname):
    
    171
    -        fullname = '%{' + varname + '}'
    
    176
    +        fullname = _wrap_variable(varname)
    
    172 177
             for key, value in _yaml.node_items(self.original):
    
    173 178
                 if fullname in value:
    
    174 179
                     provenance = _yaml.node_get_provenance(self.original, key)
    
    175 180
                     yield (key, provenance)
    
    181
    +
    
    182
    +
    
    183
    +def _wrap_variable(var):
    
    184
    +    return "%{" + var + "}"

  • buildstream/buildelement.py
    ... ... @@ -233,14 +233,13 @@ class BuildElement(Element):
    233 233
             return commands
    
    234 234
     
    
    235 235
         def __run_command(self, sandbox, cmd, cmd_name):
    
    236
    -        with self.timed_activity("Running {}".format(cmd_name)):
    
    237
    -            self.status("Running {}".format(cmd_name), detail=cmd)
    
    238
    -
    
    239
    -            # Note the -e switch to 'sh' means to exit with an error
    
    240
    -            # if any untested command fails.
    
    241
    -            #
    
    242
    -            exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    
    243
    -                                   SandboxFlags.ROOT_READ_ONLY)
    
    244
    -            if exitcode != 0:
    
    245
    -                raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    
    246
    -                                   collect=self.get_variable('install-root'))
    236
    +        self.status("Running {}".format(cmd_name), detail=cmd)
    
    237
    +
    
    238
    +        # Note the -e switch to 'sh' means to exit with an error
    
    239
    +        # if any untested command fails.
    
    240
    +        #
    
    241
    +        exitcode = sandbox.run(['sh', '-c', '-e', cmd + '\n'],
    
    242
    +                               SandboxFlags.ROOT_READ_ONLY)
    
    243
    +        if exitcode != 0:
    
    244
    +            raise ElementError("Command '{}' failed with exitcode {}".format(cmd, exitcode),
    
    245
    +                               collect=self.get_variable('install-root'))

  • buildstream/element.py
    ... ... @@ -1361,8 +1361,12 @@ class Element(Plugin):
    1361 1361
                 if not vdirectory.is_empty():
    
    1362 1362
                     raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
    
    1363 1363
     
    
    1364
    -            with tempfile.TemporaryDirectory() as temp_staging_directory:
    
    1364
    +            # While mkdtemp is advertised as using the TMP environment variable, it
    
    1365
    +            # doesn't, so this explicit extraction is necesasry.
    
    1366
    +            tmp_prefix = os.environ.get("TMP", None)
    
    1367
    +            temp_staging_directory = tempfile.mkdtemp(prefix=tmp_prefix)
    
    1365 1368
     
    
    1369
    +            try:
    
    1366 1370
                     workspace = self._get_workspace()
    
    1367 1371
                     if workspace:
    
    1368 1372
                         # If mount_workspaces is set and we're doing incremental builds,
    
    ... ... @@ -1377,6 +1381,19 @@ class Element(Plugin):
    1377 1381
                             source._stage(temp_staging_directory)
    
    1378 1382
     
    
    1379 1383
                     vdirectory.import_files(temp_staging_directory)
    
    1384
    +
    
    1385
    +            finally:
    
    1386
    +                # Staging may produce directories with less than 'rwx' permissions
    
    1387
    +                # for the owner, which will break tempfile, so we need to use chmod
    
    1388
    +                # occasionally.
    
    1389
    +                def make_dir_writable(fn, path, excinfo):
    
    1390
    +                    os.chmod(os.path.dirname(path), 0o777)
    
    1391
    +                    if os.path.isdir(path):
    
    1392
    +                        os.rmdir(path)
    
    1393
    +                    else:
    
    1394
    +                        os.remove(path)
    
    1395
    +                shutil.rmtree(temp_staging_directory, onerror=make_dir_writable)
    
    1396
    +
    
    1380 1397
             # Ensure deterministic mtime of sources at build time
    
    1381 1398
             vdirectory.set_deterministic_mtime()
    
    1382 1399
             # Ensure deterministic owners of sources at build time
    

  • doc/bst2html.py
    ... ... @@ -29,7 +29,7 @@ import sys
    29 29
     import re
    
    30 30
     import shlex
    
    31 31
     import subprocess
    
    32
    -from collections import Mapping
    
    32
    +from collections.abc import Mapping
    
    33 33
     from contextlib import contextmanager
    
    34 34
     from tempfile import TemporaryDirectory
    
    35 35
     
    

  • doc/source/developing/workspaces.rst
    ... ... @@ -4,9 +4,9 @@
    4 4
     
    
    5 5
     Workspaces
    
    6 6
     ==========
    
    7
    -
    
    8 7
     In this section we will cover the use of BuildStream's workspaces feature when devloping on a 
    
    9 8
     BuildStream project.
    
    9
    +
    
    10 10
     .. note::
    
    11 11
     
    
    12 12
        This example is distributed with BuildStream
    
    ... ... @@ -25,7 +25,6 @@ make changes to the source code of Buildstream elements by making use of
    25 25
     BuildStream's workspace command.
    
    26 26
     
    
    27 27
     
    
    28
    -
    
    29 28
     Opening a workspace
    
    30 29
     -------------------
    
    31 30
     First we need to open a workspace, we can do this by running
    
    ... ... @@ -37,12 +36,12 @@ This command has created the workspace_hello directory in which you can see
    37 36
     the source for the hello.bst element, i.e. hello.c and the corresponding
    
    38 37
     makefile.
    
    39 38
     
    
    40
    -
    
    41 39
     You can view existing workspaces using
    
    42 40
     
    
    43 41
     .. raw:: html
    
    44 42
        :file: ../sessions/developing-workspace-list.html
    
    45 43
     
    
    44
    +
    
    46 45
     Making code changes
    
    47 46
     -------------------
    
    48 47
     Let's say we want to alter the message printed when the hello command is run.
    
    ... ... @@ -66,6 +65,7 @@ This gives us the new message we changed in hello.c.
    66 65
     From this point we have several options. If the source is under version control
    
    67 66
     we can commit our changes and push them to the remote repository.
    
    68 67
     
    
    68
    +
    
    69 69
     Closing your workspace
    
    70 70
     ----------------------
    
    71 71
     If we want to close the workspace and come back to our changes later, we can
    

  • doc/source/install_linux_distro.rst
    1 1
     
    
    2
    -.. _install:
    
    3
    -
    
    4
    -Installing BuildStream on a Linux distro
    
    5
    -========================================
    
    6
    -BuildStream requires the following base system requirements:
    
    7
    -
    
    8
    -* python3 >= 3.5
    
    9
    -* bubblewrap >= 0.1.2
    
    10
    -* fuse2
    
    11
    -
    
    12
    -BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    
    13
    -Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    
    14
    -requirements of specific plugins.
    
    15
    -
    
    16
    -The default plugins with extra host dependencies are:
    
    17
    -
    
    18
    -* bzr
    
    19
    -* deb
    
    20
    -* git
    
    21
    -* ostree
    
    22
    -* patch
    
    23
    -* tar
    
    24
    -
    
    25
    -If you intend to push built artifacts to a remote artifact server,
    
    26
    -which requires special permissions, you will also need:
    
    27
    -
    
    28
    -* ssh
    
    29
    -
    
    30
    -
    
    31
    -Installing from source (recommended)
    
    32
    -------------------------------------
    
    33
    -Until BuildStream is available in your distro, you will need to install
    
    34
    -it yourself from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    
    35
    -using python's ``pip`` package manager.
    
    36
    -
    
    37
    -For the purpose of installing BuildStream while there are no distro packages,
    
    38
    -you will additionally need:
    
    39
    -
    
    40
    -* pip for python3 (only required for setup)
    
    41
    -* Python 3 development libraries and headers
    
    42
    -* git (to checkout buildstream)
    
    43
    -
    
    44
    -
    
    45
    -Installing dependencies
    
    46
    -~~~~~~~~~~~~~~~~~~~~~~~
    
    47
    -
    
    48
    -
    
    49
    -Arch Linux
    
    50
    -++++++++++
    
    51
    -Install the dependencies with::
    
    52
    -
    
    53
    -  sudo pacman -S \
    
    54
    -      python fuse2 bubblewrap \
    
    55
    -      python-pip git
    
    56
    -
    
    57
    -For the default plugins::
    
    58
    -
    
    59
    -  sudo pacman -S \
    
    60
    -      bzr git lzip ostree patch python-gobject
    
    61
    -
    
    62
    -
    
    63
    -The package *python-arpy* is required by the deb source plugin. This is not
    
    64
    -obtainable via `pacman`, you must get *python-arpy* from AUR:
    
    65
    -https://aur.archlinux.org/packages/python-arpy/
    
    66
    -
    
    67
    -To install::
    
    68
    -
    
    69
    -  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    
    70
    -  tar -xvf python-arpy.tar.gz
    
    71
    -  cd python-arpy
    
    72
    -  makepkg -si
    
    73
    -
    
    74
    -Debian
    
    75
    -++++++
    
    76
    -Install the dependencies with::
    
    77
    -
    
    78
    -  sudo apt-get install \
    
    79
    -      python3 fuse bubblewrap \
    
    80
    -      python3-pip python3-dev git
    
    81
    -
    
    82
    -For the default plugins:
    
    83
    -
    
    84
    -Stretch
    
    85
    -^^^^^^^
    
    86
    -With stretch, you first need to ensure that you have the backports repository
    
    87
    -setup as described `here <https://backports.debian.org/Instructions/>`_
    
    88
    -
    
    89
    -By adding the following line to your sources.list::
    
    90
    -
    
    91
    -  deb http://deb.debian.org/debian stretch-backports main
    
    92
    -
    
    93
    -And then running::
    
    94
    -
    
    95
    -  sudo apt update
    
    96
    -
    
    97
    -At this point you should be able to get the system requirements for the default plugins with::
    
    98
    -
    
    99
    -  sudo apt install \
    
    100
    -      bzr git lzip patch python3-arpy python3-gi
    
    101
    -  sudo apt install -t stretch-backports \
    
    102
    -      gir1.2-ostree-1.0 ostree
    
    103
    -
    
    104
    -Buster or Sid
    
    105
    -^^^^^^^^^^^^^
    
    106
    -For debian unstable or testing, only the following line should be enough
    
    107
    -to get the system requirements for the default plugins installed::
    
    108
    -
    
    109
    -  sudo apt-get install \
    
    110
    -      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    
    111
    -
    
    112
    -
    
    113
    -Fedora
    
    114
    -++++++
    
    115
    -For recent fedora systems, the following line should get you the system
    
    116
    -requirements you need::
    
    117
    -
    
    118
    -  dnf install -y \
    
    119
    -      python3 fuse bubblewrap \
    
    120
    -      python3-pip python3-devel git
    
    121
    -
    
    122
    -For the default plugins::
    
    123
    -
    
    124
    -  dnf install -y \
    
    125
    -      bzr git lzip patch ostree python3-gobject
    
    126
    -  pip3 install --user arpy
    
    127
    -
    
    128
    -
    
    129
    -Ubuntu
    
    130
    -++++++
    
    131
    -
    
    132
    -Ubuntu 18.04 LTS or later
    
    133
    -^^^^^^^^^^^^^^^^^^^^^^^^^
    
    134
    -Install the dependencies with::
    
    135
    -
    
    136
    -  sudo apt install \
    
    137
    -      python3 fuse bubblewrap \
    
    138
    -      python3-pip python3-dev git
    
    139
    -
    
    140
    -For the default plugins::
    
    141
    -
    
    142
    -  sudo apt install \
    
    143
    -      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    
    144
    -
    
    145
    -Ubuntu 16.04 LTS
    
    146
    -^^^^^^^^^^^^^^^^
    
    147
    -On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    
    148
    -or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    
    149
    -You will need to install them in whichever way you see fit. Refer the the upstream documentation
    
    150
    -for advice on this.
    
    151
    -
    
    152
    -
    
    153
    -Installing
    
    154
    -~~~~~~~~~~
    
    155
    -Once you have the base system dependencies, you can clone the BuildStream
    
    156
    -git repository and install it as a regular user::
    
    157
    -
    
    158
    -  git clone https://gitlab.com/BuildStream/buildstream.git
    
    159
    -  cd buildstream
    
    160
    -  pip3 install --user -e .
    
    161
    -
    
    162
    -This will install buildstream's pure python dependencies into
    
    163
    -your user's homedir in ``~/.local`` and will run BuildStream directly
    
    164
    -from the git checkout directory.
    
    165
    -
    
    166
    -Keep following the instructions below to ensure that the ``bst``
    
    167
    -command is in your ``PATH`` and to enable bash completions for it.
    
    168
    -
    
    169
    -.. note::
    
    170
    -
    
    171
    -   We recommend the ``-e`` option because you can upgrade your
    
    172
    -   installation by simply updating the checked out git repository.
    
    173
    -
    
    174
    -   If you want a full installation that is not linked to your
    
    175
    -   git checkout, just omit the ``-e`` option from the above commands.
    
    176
    -
    
    177
    -
    
    178
    -Adjust PATH
    
    179
    -~~~~~~~~~~~
    
    180
    -Since BuildStream is now installed under your local user's install directories,
    
    181
    -you need to ensure that ``PATH`` is adjusted.
    
    182
    -
    
    183
    -A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    
    184
    -
    
    185
    -  export PATH="${PATH}:${HOME}/.local/bin"
    
    186
    -
    
    187
    -.. note::
    
    188
    -
    
    189
    -   You will have to restart your terminal in order for these changes to take effect.
    
    190
    -
    
    191
    -
    
    192
    -Bash completions
    
    193
    -~~~~~~~~~~~~~~~~
    
    194
    -Bash completions are supported by sourcing the ``buildstream/data/bst``
    
    195
    -script found in the BuildStream repository. On many systems this script
    
    196
    -can be installed into a completions directory but when installing BuildStream
    
    197
    -without a package manager this is not an option.
    
    198
    -
    
    199
    -To enable completions for an installation of BuildStream you
    
    200
    -installed yourself from git, just append the script verbatim
    
    201
    -to your ``~/.bash_completion``:
    
    202
    -
    
    203
    -.. literalinclude:: ../../buildstream/data/bst
    
    204
    -   :language: yaml
    
    205
    -
    
    206
    -
    
    207
    -Upgrading BuildStream
    
    208
    -~~~~~~~~~~~~~~~~~~~~~
    
    209
    -Assuming you have followed the default instructions above, all
    
    210
    -you need to do to upgrade BuildStream is to update your local git
    
    211
    -checkout::
    
    212
    -
    
    213
    -  cd /path/to/buildstream
    
    214
    -  git pull --rebase
    
    215
    -
    
    216
    -If you did not specify the ``-e`` option at install time or the dependancies
    
    217
    -have changed, you will need to cleanly reinstall BuildStream::
    
    218
    -
    
    219
    -  pip3 uninstall buildstream
    
    220
    -  cd /path/to/buildstream
    
    221
    -  git pull --rebase
    
    222
    -  pip3 install --user .
    
    223 2
     
    
    3
    +.. _install_linux_distro:
    
    224 4
     
    
    225 5
     Installing from distro packages
    
    226
    --------------------------------
    
    6
    +===============================
    
    7
    +BuildStream is available on some linux distributions, here are
    
    8
    +some install instructions for the linux distributions which
    
    9
    +have packaged BuildStream.
    
    227 10
     
    
    228 11
     
    
    229 12
     Arch Linux
    
    230
    -~~~~~~~~~~
    
    13
    +----------
    
    231 14
     Packages for Arch exist in `AUR <https://wiki.archlinux.org/index.php/Arch_User_Repository#Installing_packages>`_.
    
    232 15
     Two different package versions are available:
    
    233 16
     
    
    ... ... @@ -236,8 +19,7 @@ Two different package versions are available:
    236 19
     
    
    237 20
     
    
    238 21
     Fedora
    
    239
    -~~~~~~
    
    240
    -
    
    22
    +------
    
    241 23
     BuildStream is not yet in the official Fedora repositories, but you can
    
    242 24
     install it from a Copr::
    
    243 25
     
    

  • doc/source/install_source.rst
    1
    +
    
    2
    +
    
    3
    +Installing from source
    
    4
    +======================
    
    5
    +Until BuildStream is available in :ref:`your distro <install_linux_distro>`, you will
    
    6
    +need to install it yourself from source.
    
    7
    +
    
    8
    +
    
    9
    +Installing dependencies
    
    10
    +-----------------------
    
    11
    +Before installing BuildStream from source, it is necessary to first install
    
    12
    +the system dependencies. Below are some linux distribution specific instructions
    
    13
    +for installing these dependencies.
    
    14
    +
    
    15
    +BuildStream requires the following base system requirements:
    
    16
    +
    
    17
    +* python3 >= 3.5
    
    18
    +* bubblewrap >= 0.1.2
    
    19
    +* fuse2
    
    20
    +
    
    21
    +BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
    
    22
    +Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
    
    23
    +requirements of specific plugins.
    
    24
    +
    
    25
    +The default plugins with extra host dependencies are:
    
    26
    +
    
    27
    +* bzr
    
    28
    +* deb
    
    29
    +* git
    
    30
    +* ostree
    
    31
    +* patch
    
    32
    +* tar
    
    33
    +
    
    34
    +If you intend to push built artifacts to a remote artifact server,
    
    35
    +which requires special permissions, you will also need:
    
    36
    +
    
    37
    +* ssh
    
    38
    +
    
    39
    +
    
    40
    +Arch Linux
    
    41
    +~~~~~~~~~~
    
    42
    +Install the dependencies with::
    
    43
    +
    
    44
    +  sudo pacman -S \
    
    45
    +      python fuse2 bubblewrap \
    
    46
    +      python-pip
    
    47
    +
    
    48
    +For the default plugins::
    
    49
    +
    
    50
    +  sudo pacman -S \
    
    51
    +      bzr git lzip ostree patch python-gobject
    
    52
    +
    
    53
    +
    
    54
    +The package *python-arpy* is required by the deb source plugin. This is not
    
    55
    +obtainable via `pacman`, you must get *python-arpy* from AUR:
    
    56
    +https://aur.archlinux.org/packages/python-arpy/
    
    57
    +
    
    58
    +To install::
    
    59
    +
    
    60
    +  wget https://aur.archlinux.org/cgit/aur.git/snapshot/python-arpy.tar.gz
    
    61
    +  tar -xvf python-arpy.tar.gz
    
    62
    +  cd python-arpy
    
    63
    +  makepkg -si
    
    64
    +
    
    65
    +
    
    66
    +Debian
    
    67
    +~~~~~~
    
    68
    +Install the dependencies with::
    
    69
    +
    
    70
    +  sudo apt-get install \
    
    71
    +      python3 fuse bubblewrap \
    
    72
    +      python3-pip python3-dev
    
    73
    +
    
    74
    +For the default plugins:
    
    75
    +
    
    76
    +
    
    77
    +Stretch
    
    78
    ++++++++
    
    79
    +With stretch, you first need to ensure that you have the backports repository
    
    80
    +setup as described `here <https://backports.debian.org/Instructions/>`_
    
    81
    +
    
    82
    +By adding the following line to your sources.list::
    
    83
    +
    
    84
    +  deb http://deb.debian.org/debian stretch-backports main
    
    85
    +
    
    86
    +And then running::
    
    87
    +
    
    88
    +  sudo apt update
    
    89
    +
    
    90
    +At this point you should be able to get the system requirements for the default plugins with::
    
    91
    +
    
    92
    +  sudo apt install \
    
    93
    +      bzr git lzip patch python3-arpy python3-gi
    
    94
    +  sudo apt install -t stretch-backports \
    
    95
    +      gir1.2-ostree-1.0 ostree
    
    96
    +
    
    97
    +
    
    98
    +Buster or Sid
    
    99
    ++++++++++++++
    
    100
    +For debian unstable or testing, only the following line should be enough
    
    101
    +to get the system requirements for the default plugins installed::
    
    102
    +
    
    103
    +  sudo apt-get install \
    
    104
    +      lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
    
    105
    +
    
    106
    +
    
    107
    +Fedora
    
    108
    +~~~~~~
    
    109
    +For recent fedora systems, the following line should get you the system
    
    110
    +requirements you need::
    
    111
    +
    
    112
    +  dnf install -y \
    
    113
    +      python3 fuse bubblewrap \
    
    114
    +      python3-pip python3-devel
    
    115
    +
    
    116
    +For the default plugins::
    
    117
    +
    
    118
    +  dnf install -y \
    
    119
    +      bzr git lzip patch ostree python3-gobject
    
    120
    +  pip3 install --user arpy
    
    121
    +
    
    122
    +
    
    123
    +Ubuntu
    
    124
    +~~~~~~
    
    125
    +
    
    126
    +
    
    127
    +Ubuntu 18.04 LTS or later
    
    128
    ++++++++++++++++++++++++++
    
    129
    +Install the dependencies with::
    
    130
    +
    
    131
    +  sudo apt install \
    
    132
    +      python3 fuse bubblewrap \
    
    133
    +      python3-pip python3-dev
    
    134
    +
    
    135
    +For the default plugins::
    
    136
    +
    
    137
    +  sudo apt install \
    
    138
    +      bzr gir1.2-ostree-1.0 git lzip ostree patch python3-arpy python3-gi
    
    139
    +
    
    140
    +
    
    141
    +Ubuntu 16.04 LTS
    
    142
    +++++++++++++++++
    
    143
    +On Ubuntu 16.04, neither `bubblewrap <https://github.com/projectatomic/bubblewrap/>`_
    
    144
    +or `ostree <https://github.com/ostreedev/ostree>`_ are available in the official repositories.
    
    145
    +You will need to install them in whichever way you see fit. Refer the the upstream documentation
    
    146
    +for advice on this.
    
    147
    +
    
    148
    +
    
    149
    +Installing
    
    150
    +----------
    
    151
    +Once you have the base system dependencies, you can install the BuildStream
    
    152
    +python package as a regular user.
    
    153
    +
    
    154
    +
    
    155
    +Installing from PyPI (recommended)
    
    156
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    157
    +Since we only ever publish :ref:`release versions <install_semantic_versioning>` on
    
    158
    +PyPI, it is currently recommended to use this installation path. This will
    
    159
    +ensure that you always have the latest recommended version of BuildStream that
    
    160
    +we recommend.
    
    161
    +
    
    162
    +To install from PyPI, you will additionally require:
    
    163
    +
    
    164
    +* pip for python3 (only required for setup)
    
    165
    +* Python 3 development libraries and headers
    
    166
    +
    
    167
    +Simply run the following command::
    
    168
    +
    
    169
    +  pip3 install --user BuildStream
    
    170
    +
    
    171
    +This will install latest stable version of BuildStream and its pure python
    
    172
    +dependencies into your user's homedir in ``~/.local``.
    
    173
    +
    
    174
    +Keep following the instructions below to ensure that the ``bst``
    
    175
    +command is in your ``PATH`` and to enable bash completions for it.
    
    176
    +
    
    177
    +.. note::
    
    178
    +
    
    179
    +  If you want a specific version of BuildStream, you can install it using
    
    180
    +  ``pip install --user BuildStream==<version-number>``
    
    181
    +
    
    182
    +
    
    183
    +Upgrading from PyPI
    
    184
    ++++++++++++++++++++
    
    185
    +Once you have already installed BuildStream from PyPI, you can later update
    
    186
    +to the latest recommended version like so::
    
    187
    +
    
    188
    +  pip install --user --upgrade BuildStream
    
    189
    +
    
    190
    +
    
    191
    +.. _install_git_checkout:
    
    192
    +
    
    193
    +Installing from a git checkout
    
    194
    +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
    
    195
    +To install directly from the `git repository <https://gitlab.com/BuildStream/buildstream.git>`_
    
    196
    +using python's ``pip`` package manager, you will additionally require:
    
    197
    +
    
    198
    +* pip for python3 (only required for setup)
    
    199
    +* Python 3 development libraries and headers
    
    200
    +* git (to checkout BuildStream)
    
    201
    +
    
    202
    +Before installing, please check the existing tags in the git repository
    
    203
    +and determine which version you want to install, and whether you want
    
    204
    +to install an official release version (recommended), or a development snapshot
    
    205
    +to help us out testing the bleeding edge of development. Follow the
    
    206
    +:ref:`semantic versioning guide <install_semantic_versioning>` to determine
    
    207
    +which tag you intend to install.
    
    208
    +
    
    209
    +Run the following commands::
    
    210
    +
    
    211
    +  git clone https://gitlab.com/BuildStream/buildstream.git
    
    212
    +  cd buildstream
    
    213
    +  git checkout <desired release tag>
    
    214
    +  pip3 install --user -e .
    
    215
    +
    
    216
    +This will install buildstream's pure python dependencies into
    
    217
    +your user's homedir in ``~/.local`` and will run BuildStream directly
    
    218
    +from the git checkout directory.
    
    219
    +
    
    220
    +Keep following the instructions below to ensure that the ``bst``
    
    221
    +command is in your ``PATH`` and to enable bash completions for it.
    
    222
    +
    
    223
    +.. note::
    
    224
    +
    
    225
    +   We recommend the ``-e`` option because you can upgrade your
    
    226
    +   installation by simply updating the checked out git repository.
    
    227
    +
    
    228
    +   If you want a full installation that is not linked to your
    
    229
    +   git checkout, just omit the ``-e`` option from the above commands.
    
    230
    +
    
    231
    +
    
    232
    +Upgrading from a git checkout
    
    233
    ++++++++++++++++++++++++++++++
    
    234
    +If you installed BuildStream from a local git checkout using ``-e`` option, all
    
    235
    +you need to do to upgrade BuildStream is to update your local git checkout::
    
    236
    +
    
    237
    +  cd /path/to/buildstream
    
    238
    +  git pull --rebase
    
    239
    +
    
    240
    +If you did not specify the ``-e`` option at install time or the dependancies
    
    241
    +have changed, you will need to cleanly reinstall BuildStream::
    
    242
    +
    
    243
    +  pip3 uninstall buildstream
    
    244
    +  cd /path/to/buildstream
    
    245
    +  git pull --rebase
    
    246
    +  pip3 install --user .
    
    247
    +
    
    248
    +.. note::
    
    249
    +
    
    250
    +   If BuildStream has added any dependencies since the last upgrade,
    
    251
    +   you will need to uninstall and reinstall to ensure those dependencies
    
    252
    +   are met, regardless of whether you have used the ``-e`` option at
    
    253
    +   install time.
    
    254
    +
    
    255
    +
    
    256
    +Post install setup
    
    257
    +------------------
    
    258
    +After having installed from source using any of the above methods, some
    
    259
    +setup will be required to use BuildStream.
    
    260
    +
    
    261
    +
    
    262
    +Adjust PATH
    
    263
    +~~~~~~~~~~~
    
    264
    +Since BuildStream is now installed under your local user's install directories,
    
    265
    +you need to ensure that ``PATH`` is adjusted.
    
    266
    +
    
    267
    +A regular way to do this is to add the following line to the end of your ``~/.bashrc``::
    
    268
    +
    
    269
    +  export PATH="${PATH}:${HOME}/.local/bin"
    
    270
    +
    
    271
    +.. note::
    
    272
    +
    
    273
    +   You will have to restart your terminal in order for these changes to take effect.
    
    274
    +
    
    275
    +
    
    276
    +Bash completions
    
    277
    +~~~~~~~~~~~~~~~~
    
    278
    +Bash completions are supported by sourcing the ``buildstream/data/bst``
    
    279
    +script found in the BuildStream repository. On many systems this script
    
    280
    +can be installed into a completions directory but when installing BuildStream
    
    281
    +without a package manager this is not an option.
    
    282
    +
    
    283
    +To enable completions for an installation of BuildStream you
    
    284
    +installed yourself from git, just append the script verbatim
    
    285
    +to your ``~/.bash_completion``:
    
    286
    +
    
    287
    +.. literalinclude:: ../../buildstream/data/bst
    
    288
    +   :language: yaml

  • doc/source/install_versions.rst
    1
    +
    
    2
    +
    
    3
    +.. _install_semantic_versioning:
    
    4
    +
    
    5
    +Semantic Versioning
    
    6
    +===================
    
    7
    +BuildStream follows the Semantic Versioning Convention `(SemVer) <https://semver.org/>`_,
    
    8
    +and uses even minor point numbers to denote releases intended for users while
    
    9
    +odd minor point numbers represent development snapshops.
    
    10
    +
    
    11
    +For example, for a given version number ``X.Y.Z``
    
    12
    + * The ``X.<even number>.*`` versions are releases intended for users.
    
    13
    + * The ``X.<odd number>.*`` versions are development spanshots intended for testing.
    
    14
    +
    
    15
    +If you are :ref:`installing from git <install_git_checkout>`, please look for the latest
    
    16
    +tag in the latest release branch to ensure you're getting the latest release.
    
    17
    +
    
    18
    +Current release branches:
    
    19
    + * `bst-1.2 (latest) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.2>`_
    
    20
    + * `bst-1.0 (deprecated) <https://gitlab.com/BuildStream/buildstream/commits/bst-1.0>`_

  • doc/source/main_install.rst
    1
    +
    
    2
    +
    
    3
    +.. _install:
    
    4
    +
    
    1 5
     Install
    
    2 6
     =======
    
    3
    -This section covers how to install BuildStream onto your machine, how to run
    
    4
    -BuildStream inside a docker image and also how to configure an artifact server.
    
    7
    +This section provides instructions for installing BuildStream and its
    
    8
    +companion artifact server on various platforms, along with any installation
    
    9
    +related materials.
    
    5 10
     
    
    6 11
     .. note::
    
    7 12
     
    
    8
    -   BuildStream is not currently supported natively on macOS and Windows. Windows
    
    9
    -   and macOS users should refer to :ref:`docker`.
    
    13
    +   BuildStream is currently only supported natively on Linux. Users of Unix-like
    
    14
    +   systems where Docker is available can still use BuildStream by following the
    
    15
    +   :ref:`Docker install guide <docker>`
    
    10 16
     
    
    11 17
     .. toctree::
    
    12
    -   :maxdepth: 2
    
    18
    +   :maxdepth: 1
    
    13 19
     
    
    20
    +   install_source
    
    14 21
        install_linux_distro
    
    15 22
        install_docker
    
    16 23
        install_artifacts
    
    24
    +   install_versions

  • doc/source/sessions/developing-build-after-changes.htmldoc/source/sessions-stored/developing-build-after-changes.html

  • doc/source/sessions/developing-close-workspace.htmldoc/source/sessions-stored/developing-close-workspace.html

  • doc/source/sessions/developing-discard-workspace.htmldoc/source/sessions-stored/developing-discard-workspace.html

  • doc/source/sessions/developing-reopen-workspace.htmldoc/source/sessions-stored/developing-reopen-workspace.html

  • doc/source/sessions/developing-reset-workspace.htmldoc/source/sessions-stored/developing-reset-workspace.html

  • doc/source/sessions/developing-shell-after-changes.htmldoc/source/sessions-stored/developing-shell-after-changes.html

  • doc/source/sessions/developing-workspace-list.htmldoc/source/sessions-stored/developing-workspace-list.html

  • doc/source/sessions/developing-workspace-open.htmldoc/source/sessions-stored/developing-workspace-open.html

  • setup.cfg
    ... ... @@ -11,7 +11,7 @@ parentdir_prefix = BuildStream-
    11 11
     test=pytest
    
    12 12
     
    
    13 13
     [tool:pytest]
    
    14
    -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc
    
    14
    +addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
    
    15 15
     norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
    
    16 16
     python_files = tests/*/*.py
    
    17 17
     pep8maxlinelength = 119
    

  • tests/integration/project/elements/base/base-alpine.bst
    ... ... @@ -7,6 +7,6 @@ description: |
    7 7
     
    
    8 8
     sources:
    
    9 9
       - kind: tar
    
    10
    -    url: sysroot:tarballs/integration-tests-base.v1.x86_64.tar.xz
    
    10
    +    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    
    11 11
         base-dir: ''
    
    12 12
         ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

  • tests/integration/project/project.conf
    ... ... @@ -2,7 +2,7 @@
    2 2
     name: test
    
    3 3
     element-path: elements
    
    4 4
     aliases:
    
    5
    -  sysroot: https://gnome7.codethink.co.uk/
    
    5
    +  alpine: https://gnome7.codethink.co.uk/tarballs/
    
    6 6
       project_dir: file://{project_dir}
    
    7 7
     options:
    
    8 8
       linux:
    

  • tests/integration/source-determinism.py
    ... ... @@ -2,7 +2,8 @@ import os
    2 2
     import pytest
    
    3 3
     
    
    4 4
     from buildstream import _yaml, utils
    
    5
    -from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    
    5
    +from tests.testutils import create_repo, ALL_REPO_KINDS
    
    6
    +from tests.testutils import cli_integration as cli
    
    6 7
     
    
    7 8
     
    
    8 9
     DATA_DIR = os.path.join(
    
    ... ... @@ -28,7 +29,7 @@ def create_test_directory(*path, mode=0o644):
    28 29
     @pytest.mark.integration
    
    29 30
     @pytest.mark.datafiles(DATA_DIR)
    
    30 31
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    
    31
    -def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    
    32
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_cache):
    
    32 33
         project = str(datafiles)
    
    33 34
         element_name = 'list'
    
    34 35
         element_path = os.path.join(project, 'elements', element_name)
    
    ... ... @@ -91,14 +92,16 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    91 92
                     return f.read()
    
    92 93
             finally:
    
    93 94
                 os.umask(old_umask)
    
    94
    -            cli.remove_artifact_from_cache(project, element_name)
    
    95
    +            cache_dir = os.path.join(integration_cache, 'artifacts')
    
    96
    +            cli.remove_artifact_from_cache(project, element_name,
    
    97
    +                                           cache_dir=cache_dir)
    
    95 98
     
    
    96 99
         assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    
    97 100
     
    
    98 101
     
    
    99 102
     @pytest.mark.integration
    
    100 103
     @pytest.mark.datafiles(DATA_DIR)
    
    101
    -def test_deterministic_source_local(cli, tmpdir, datafiles):
    
    104
    +def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
    
    102 105
         """Only user rights should be considered for local source.
    
    103 106
         """
    
    104 107
         project = str(datafiles)
    
    ... ... @@ -150,6 +153,8 @@ def test_deterministic_source_local(cli, tmpdir, datafiles):
    150 153
                 with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    151 154
                     return f.read()
    
    152 155
             finally:
    
    153
    -            cli.remove_artifact_from_cache(project, element_name)
    
    156
    +            cache_dir = os.path.join(integration_cache, 'artifacts')
    
    157
    +            cli.remove_artifact_from_cache(project, element_name,
    
    158
    +                                           cache_dir=cache_dir)
    
    154 159
     
    
    155 160
         assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

  • tests/sources/tar.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
     import tarfile
    
    4 4
     import tempfile
    
    5 5
     import subprocess
    
    6
    +from shutil import copyfile, rmtree
    
    6 7
     
    
    7 8
     from buildstream._exceptions import ErrorDomain
    
    8 9
     from buildstream import _yaml
    
    ... ... @@ -257,3 +258,47 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
    257 258
         original_contents = list_dir_contents(original_dir)
    
    258 259
         checkout_contents = list_dir_contents(checkoutdir)
    
    259 260
         assert(checkout_contents == original_contents)
    
    261
    +
    
    262
    +
    
    263
    +# Test that a tarball that contains a read only dir works
    
    264
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only'))
    
    265
    +def test_read_only_dir(cli, tmpdir, datafiles):
    
    266
    +    try:
    
    267
    +        project = os.path.join(datafiles.dirname, datafiles.basename)
    
    268
    +        generate_project(project, tmpdir)
    
    269
    +
    
    270
    +        # Get the tarball in tests/sources/tar/read-only/content
    
    271
    +        #
    
    272
    +        # NOTE that we need to do this because tarfile.open and tar.add()
    
    273
    +        # are packing the tar up with writeable files and dirs
    
    274
    +        tarball = os.path.join(str(datafiles), 'content', 'a.tar.gz')
    
    275
    +        if not os.path.exists(tarball):
    
    276
    +            raise FileNotFoundError('{} does not exist'.format(tarball))
    
    277
    +        copyfile(tarball, os.path.join(str(tmpdir), 'a.tar.gz'))
    
    278
    +
    
    279
    +        # Because this test can potentially leave directories behind
    
    280
    +        # which are difficult to remove, ask buildstream to use
    
    281
    +        # our temp directory, so we can clean up.
    
    282
    +        tmpdir_str = str(tmpdir)
    
    283
    +        if not tmpdir_str.endswith(os.path.sep):
    
    284
    +            tmpdir_str += os.path.sep
    
    285
    +        env = {"TMP": tmpdir_str}
    
    286
    +
    
    287
    +        # Track, fetch, build, checkout
    
    288
    +        result = cli.run(project=project, args=['track', 'target.bst'], env=env)
    
    289
    +        result.assert_success()
    
    290
    +        result = cli.run(project=project, args=['fetch', 'target.bst'], env=env)
    
    291
    +        result.assert_success()
    
    292
    +        result = cli.run(project=project, args=['build', 'target.bst'], env=env)
    
    293
    +        result.assert_success()
    
    294
    +
    
    295
    +    finally:
    
    296
    +
    
    297
    +        # Make tmpdir deletable no matter what happens
    
    298
    +        def make_dir_writable(fn, path, excinfo):
    
    299
    +            os.chmod(os.path.dirname(path), 0o777)
    
    300
    +            if os.path.isdir(path):
    
    301
    +                os.rmdir(path)
    
    302
    +            else:
    
    303
    +                os.remove(path)
    
    304
    +        rmtree(str(tmpdir), onerror=make_dir_writable)

  • tests/sources/tar/read-only/content/a.tar.gz
    No preview for this file type
  • tests/sources/tar/read-only/target.bst
    1
    +kind: import
    
    2
    +description: The kind of this element is irrelevant.
    
    3
    +sources:
    
    4
    +- kind: tar
    
    5
    +  url: tmpdir:/a.tar.gz
    
    6
    +  ref: foo

  • tests/testutils/runcli.py
    ... ... @@ -198,8 +198,10 @@ class Cli():
    198 198
             for key, val in config.items():
    
    199 199
                 self.config[key] = val
    
    200 200
     
    
    201
    -    def remove_artifact_from_cache(self, project, element_name):
    
    202
    -        cache_dir = os.path.join(project, 'cache', 'artifacts')
    
    201
    +    def remove_artifact_from_cache(self, project, element_name,
    
    202
    +                                   *, cache_dir=None):
    
    203
    +        if not cache_dir:
    
    204
    +            cache_dir = os.path.join(project, 'cache', 'artifacts')
    
    203 205
     
    
    204 206
             cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
    
    205 207
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]