Ed Baunton pushed to branch edbaunton/remote-source at BuildStream / buildstream
Commits:
-
60290f31
by Javier Jardón at 2018-07-23T15:29:37Z
-
de162e43
by Javier Jardón at 2018-07-24T11:58:16Z
-
6106d657
by Josh Smith at 2018-07-25T10:50:14Z
-
4a637d63
by Jonathan Maw at 2018-07-25T12:05:19Z
-
a0ad985f
by Phil Dawson at 2018-07-25T13:03:23Z
-
95b4eae4
by Phil Dawson at 2018-07-25T13:03:23Z
-
889bf238
by Jim MacArthur at 2018-07-25T13:43:21Z
-
54fee6c8
by Phillip Smyth at 2018-07-25T14:15:36Z
-
f62b6cb7
by Jonathan Maw at 2018-07-25T15:01:33Z
-
c049d021
by Ed Baunton at 2018-07-25T19:43:51Z
22 changed files:
- buildstream/_artifactcache/cascache.py
- buildstream/_frontend/app.py
- buildstream/_stream.py
- buildstream/_versions.py
- + buildstream/plugins/sources/remote.py
- doc/source/core_plugins.rst
- doc/source/install_linux_distro.rst
- tests/frontend/show.py
- tests/frontend/workspace.py
- + tests/sources/remote.py
- + tests/sources/remote/missing-file/target.bst
- + tests/sources/remote/no-ref/file
- + tests/sources/remote/no-ref/target.bst
- + tests/sources/remote/path-in-filename/dir/file
- + tests/sources/remote/path-in-filename/target.bst
- + tests/sources/remote/single-file-custom-name/dir/file
- + tests/sources/remote/single-file-custom-name/target.bst
- + tests/sources/remote/single-file/dir/file
- + tests/sources/remote/single-file/target.bst
- + tests/sources/remote/unique-keys/dir/file
- + tests/sources/remote/unique-keys/target-custom.bst
- + tests/sources/remote/unique-keys/target.bst
Changes:
| ... | ... | @@ -240,7 +240,6 @@ class CASCache(ArtifactCache): |
| 240 | 240 |
|
| 241 | 241 |
except grpc.RpcError as e:
|
| 242 | 242 |
if e.code() != grpc.StatusCode.NOT_FOUND:
|
| 243 |
- element.info("{} not found at remote {}".format(element._get_brief_display_key(), remote.spec.url))
|
|
| 244 | 243 |
raise
|
| 245 | 244 |
|
| 246 | 245 |
return False
|
| ... | ... | @@ -270,6 +270,10 @@ class App(): |
| 270 | 270 |
|
| 271 | 271 |
# Exit with the error
|
| 272 | 272 |
self._error_exit(e)
|
| 273 |
+ except RecursionError:
|
|
| 274 |
+ click.echo("RecursionError: Depency depth is too large. Maximum recursion depth exceeded.",
|
|
| 275 |
+ err=True)
|
|
| 276 |
+ sys.exit(-1)
|
|
| 273 | 277 |
|
| 274 | 278 |
else:
|
| 275 | 279 |
# No exceptions occurred, print session time and summary
|
| ... | ... | @@ -476,7 +476,7 @@ class Stream(): |
| 476 | 476 |
|
| 477 | 477 |
# Check for workspace config
|
| 478 | 478 |
workspace = workspaces.get_workspace(target._get_full_name())
|
| 479 |
- if workspace:
|
|
| 479 |
+ if workspace and not force:
|
|
| 480 | 480 |
raise StreamError("Workspace '{}' is already defined at: {}"
|
| 481 | 481 |
.format(target.name, workspace.path))
|
| 482 | 482 |
|
| ... | ... | @@ -495,6 +495,10 @@ class Stream(): |
| 495 | 495 |
"fetch the latest version of the " +
|
| 496 | 496 |
"source.")
|
| 497 | 497 |
|
| 498 |
+ if workspace:
|
|
| 499 |
+ workspaces.delete_workspace(target._get_full_name())
|
|
| 500 |
+ workspaces.save_config()
|
|
| 501 |
+ shutil.rmtree(directory)
|
|
| 498 | 502 |
try:
|
| 499 | 503 |
os.makedirs(directory, exist_ok=True)
|
| 500 | 504 |
except OSError as e:
|
| ... | ... | @@ -23,7 +23,7 @@ |
| 23 | 23 |
# This version is bumped whenever enhancements are made
|
| 24 | 24 |
# to the `project.conf` format or the core element format.
|
| 25 | 25 |
#
|
| 26 |
-BST_FORMAT_VERSION = 9
|
|
| 26 |
+BST_FORMAT_VERSION = 10
|
|
| 27 | 27 |
|
| 28 | 28 |
|
| 29 | 29 |
# The base BuildStream artifact version
|
| 1 |
+#
|
|
| 2 |
+# Copyright Bloomberg Finance LP
|
|
| 3 |
+#
|
|
| 4 |
+# This program is free software; you can redistribute it and/or
|
|
| 5 |
+# modify it under the terms of the GNU Lesser General Public
|
|
| 6 |
+# License as published by the Free Software Foundation; either
|
|
| 7 |
+# version 2 of the License, or (at your option) any later version.
|
|
| 8 |
+#
|
|
| 9 |
+# This library is distributed in the hope that it will be useful,
|
|
| 10 |
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
| 11 |
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
| 12 |
+# Lesser General Public License for more details.
|
|
| 13 |
+#
|
|
| 14 |
+# You should have received a copy of the GNU Lesser General Public
|
|
| 15 |
+# License along with this library. If not, see <http://www.gnu.org/licenses/>.
|
|
| 16 |
+#
|
|
| 17 |
+# Authors:
|
|
| 18 |
+# Ed Baunton <ebaunton1 bloomberg net>
|
|
| 19 |
+ |
|
| 20 |
+"""
|
|
| 21 |
+remote - stage files from remote urls
|
|
| 22 |
+=====================================
|
|
| 23 |
+ |
|
| 24 |
+**Usage:**
|
|
| 25 |
+ |
|
| 26 |
+.. code:: yaml
|
|
| 27 |
+ |
|
| 28 |
+ # Specify the remote source kind
|
|
| 29 |
+ kind: remote
|
|
| 30 |
+ |
|
| 31 |
+ # Optionally specify a relative staging directory
|
|
| 32 |
+ # directory: path/to/stage
|
|
| 33 |
+ |
|
| 34 |
+ # Optionally specify a relative staging filename.
|
|
| 35 |
+ # If not specified, the basename of the url will be used.
|
|
| 36 |
+ # filename: customfilename
|
|
| 37 |
+ |
|
| 38 |
+ # Specify the url. Using an alias defined in your project
|
|
| 39 |
+ # configuration is encouraged. 'bst track' will update the
|
|
| 40 |
+ # sha256sum in 'ref' to the downloaded file's sha256sum.
|
|
| 41 |
+ url: upstream:foo
|
|
| 42 |
+ |
|
| 43 |
+ # Specify the ref. It's a sha256sum of the file you download.
|
|
| 44 |
+ ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
|
|
| 45 |
+ |
|
| 46 |
+.. note::
|
|
| 47 |
+ |
|
| 48 |
+ The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
|
|
| 49 |
+ |
|
| 50 |
+"""
|
|
| 51 |
+import os
|
|
| 52 |
+from buildstream import SourceError, utils
|
|
| 53 |
+from ._downloadablefilesource import DownloadableFileSource
|
|
| 54 |
+ |
|
| 55 |
+ |
|
| 56 |
+class RemoteSource(DownloadableFileSource):
|
|
| 57 |
+ # pylint: disable=attribute-defined-outside-init
|
|
| 58 |
+ |
|
| 59 |
+ def configure(self, node):
|
|
| 60 |
+ super().configure(node)
|
|
| 61 |
+ |
|
| 62 |
+ self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
|
|
| 63 |
+ |
|
| 64 |
+ if os.sep in self.filename:
|
|
| 65 |
+ raise SourceError('{}: filename parameter cannot contain directories'.format(self))
|
|
| 66 |
+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename'])
|
|
| 67 |
+ |
|
| 68 |
+ def get_unique_key(self):
|
|
| 69 |
+ return super().get_unique_key() + [self.filename]
|
|
| 70 |
+ |
|
| 71 |
+ def stage(self, directory):
|
|
| 72 |
+ # Same as in local plugin, don't use hardlinks to stage sources, they
|
|
| 73 |
+ # are not write protected in the sandbox.
|
|
| 74 |
+ dest = os.path.join(directory, self.filename)
|
|
| 75 |
+ with self.timed_activity("Staging remote file to {}".format(dest)):
|
|
| 76 |
+ utils.safe_copy(self._get_mirror_file(), dest)
|
|
| 77 |
+ |
|
| 78 |
+ |
|
| 79 |
+def setup():
|
|
| 80 |
+ return RemoteSource
|
| ... | ... | @@ -50,6 +50,7 @@ Sources |
| 50 | 50 |
:maxdepth: 1
|
| 51 | 51 |
|
| 52 | 52 |
sources/local
|
| 53 |
+ sources/remote
|
|
| 53 | 54 |
sources/tar
|
| 54 | 55 |
sources/zip
|
| 55 | 56 |
sources/git
|
| ... | ... | @@ -6,10 +6,8 @@ Installing BuildStream on a Linux distro |
| 6 | 6 |
BuildStream requires the following base system requirements:
|
| 7 | 7 |
|
| 8 | 8 |
* python3 >= 3.5
|
| 9 |
-* libostree >= v2017.8 with introspection data
|
|
| 10 | 9 |
* bubblewrap >= 0.1.2
|
| 11 | 10 |
* fuse2
|
| 12 |
-* PyGObject introspection bindings
|
|
| 13 | 11 |
|
| 14 | 12 |
BuildStream also depends on the host tools for the :mod:`Source <buildstream.source>` plugins.
|
| 15 | 13 |
Refer to the respective :ref:`source plugin <plugins_sources>` documentation for host tool
|
| ... | ... | @@ -20,6 +18,7 @@ The default plugins with extra host dependencies are: |
| 20 | 18 |
* bzr
|
| 21 | 19 |
* deb
|
| 22 | 20 |
* git
|
| 21 |
+* ostree
|
|
| 23 | 22 |
* patch
|
| 24 | 23 |
* tar
|
| 25 | 24 |
|
| ... | ... | @@ -52,18 +51,24 @@ Arch Linux |
| 52 | 51 |
Install the dependencies with::
|
| 53 | 52 |
|
| 54 | 53 |
sudo pacman -S \
|
| 55 |
- python fuse2 ostree bubblewrap python-gobject \
|
|
| 54 |
+ python fuse2 bubblewrap \
|
|
| 56 | 55 |
python-pip git
|
| 57 | 56 |
|
| 58 | 57 |
For the default plugins::
|
| 59 | 58 |
|
| 60 | 59 |
sudo pacman -S \
|
| 61 |
- lzip git bzr patch python-arpy
|
|
| 60 |
+ bzr git lzip ostree patch python-arpy python-gobject
|
|
| 62 | 61 |
|
| 63 | 62 |
|
| 64 | 63 |
Debian
|
| 65 | 64 |
++++++
|
| 65 |
+Install the dependencies with::
|
|
| 66 |
+ |
|
| 67 |
+ sudo apt-get install \
|
|
| 68 |
+ python3 fuse bubblewrap \
|
|
| 69 |
+ python3-pip python3-dev git
|
|
| 66 | 70 |
|
| 71 |
+For the default plugins:
|
|
| 67 | 72 |
|
| 68 | 73 |
Stretch
|
| 69 | 74 |
^^^^^^^
|
| ... | ... | @@ -78,32 +83,20 @@ And then running:: |
| 78 | 83 |
|
| 79 | 84 |
sudo apt-get update
|
| 80 | 85 |
|
| 81 |
-At this point you should be able to get the system requirements with::
|
|
| 86 |
+At this point you should be able to get the system requirements for the default plugins with::
|
|
| 82 | 87 |
|
| 83 | 88 |
sudo apt-get install \
|
| 84 |
- python3 fuse ostree gir1.2-ostree-1.0 bubblewrap python3-gi \
|
|
| 85 |
- python3-pip python3-dev git
|
|
| 89 |
+ bzr git lzip patch python3-arpy python3-gi
|
|
| 86 | 90 |
sudo apt-get install -t stretch-backports \
|
| 87 | 91 |
gir1.2-ostree-1.0 ostree
|
| 88 | 92 |
|
| 89 |
-For the default plugins::
|
|
| 90 |
- |
|
| 91 |
- sudo apt-get install \
|
|
| 92 |
- lzip git bzr patch python3-arpy
|
|
| 93 |
- |
|
| 94 | 93 |
Buster or Sid
|
| 95 | 94 |
^^^^^^^^^^^^^
|
| 96 | 95 |
For debian unstable or testing, only the following line should be enough
|
| 97 |
-to get the base system requirements installed::
|
|
| 98 |
- |
|
| 99 |
- sudo apt-get install \
|
|
| 100 |
- python3 fuse ostree gir1.2-ostree-1.0 bubblewrap python3-gi \
|
|
| 101 |
- python3-pip python3-dev git
|
|
| 102 |
- |
|
| 103 |
-For the default plugins::
|
|
| 96 |
+to get the system requirements for the default plugins installed::
|
|
| 104 | 97 |
|
| 105 | 98 |
sudo apt-get install \
|
| 106 |
- lzip git bzr patch python3-arpy
|
|
| 99 |
+ lzip gir1.2-ostree-1.0 git bzr ostree patch python3-arpy python3-gi
|
|
| 107 | 100 |
|
| 108 | 101 |
|
| 109 | 102 |
Fedora
|
| ... | ... | @@ -112,13 +105,13 @@ For recent fedora systems, the following line should get you the system |
| 112 | 105 |
requirements you need::
|
| 113 | 106 |
|
| 114 | 107 |
dnf install -y \
|
| 115 |
- python3 fuse ostree bubblewrap python3-gobject \
|
|
| 108 |
+ python3 fuse bubblewrap \
|
|
| 116 | 109 |
python3-pip python3-devel git
|
| 117 | 110 |
|
| 118 | 111 |
For the default plugins::
|
| 119 | 112 |
|
| 120 | 113 |
dnf install -y \
|
| 121 |
- lzip git bzr patch python3-arpy
|
|
| 114 |
+ bzr git lzip patch ostree python3-arpy python3-gobject
|
|
| 122 | 115 |
|
| 123 | 116 |
|
| 124 | 117 |
Installing
|
| 1 | 1 |
import os
|
| 2 |
-import pytest
|
|
| 2 |
+import sys
|
|
| 3 |
+import shutil
|
|
| 3 | 4 |
import itertools
|
| 5 |
+import pytest
|
|
| 4 | 6 |
from tests.testutils import cli
|
| 5 |
- |
|
| 6 | 7 |
from buildstream import _yaml
|
| 7 | 8 |
from buildstream._exceptions import ErrorDomain, LoadErrorReason
|
| 8 | 9 |
|
| ... | ... | @@ -232,3 +233,58 @@ def test_fetched_junction(cli, tmpdir, datafiles, element_name): |
| 232 | 233 |
|
| 233 | 234 |
results = result.output.strip().splitlines()
|
| 234 | 235 |
assert 'junction.bst:import-etc.bst-buildable' in results
|
| 236 |
+ |
|
| 237 |
+ |
|
| 238 |
+###############################################################
|
|
| 239 |
+# Testing recursion depth #
|
|
| 240 |
+###############################################################
|
|
| 241 |
+@pytest.mark.parametrize("dependency_depth", [100, 500, 1200])
|
|
| 242 |
+def test_exceed_max_recursion_depth(cli, tmpdir, dependency_depth):
|
|
| 243 |
+ project_name = "recursion-test"
|
|
| 244 |
+ path = str(tmpdir)
|
|
| 245 |
+ project_path = os.path.join(path, project_name)
|
|
| 246 |
+ |
|
| 247 |
+ def setup_test():
|
|
| 248 |
+ """
|
|
| 249 |
+ Creates a bst project with dependencydepth + 1 elements, each of which
|
|
| 250 |
+ depends of the previous element to be created. Each element created
|
|
| 251 |
+ is of type import and has an empty source file.
|
|
| 252 |
+ """
|
|
| 253 |
+ os.mkdir(project_path)
|
|
| 254 |
+ |
|
| 255 |
+ result = cli.run(project=project_path, silent=True,
|
|
| 256 |
+ args=['init', '--project-name', project_name])
|
|
| 257 |
+ result.assert_success()
|
|
| 258 |
+ |
|
| 259 |
+ sourcefiles_path = os.path.join(project_path, "files")
|
|
| 260 |
+ os.mkdir(sourcefiles_path)
|
|
| 261 |
+ |
|
| 262 |
+ element_path = os.path.join(project_path, "elements")
|
|
| 263 |
+ for i in range(0, dependency_depth + 1):
|
|
| 264 |
+ element = {
|
|
| 265 |
+ 'kind': 'import',
|
|
| 266 |
+ 'sources': [{'kind': 'local',
|
|
| 267 |
+ 'path': 'files/source{}'.format(str(i))}],
|
|
| 268 |
+ 'depends': ['element{}.bst'.format(str(i - 1))]
|
|
| 269 |
+ }
|
|
| 270 |
+ if i == 0:
|
|
| 271 |
+ del element['depends']
|
|
| 272 |
+ _yaml.dump(element, os.path.join(element_path, "element{}.bst".format(str(i))))
|
|
| 273 |
+ |
|
| 274 |
+ source = os.path.join(sourcefiles_path, "source{}".format(str(i)))
|
|
| 275 |
+ open(source, 'x').close()
|
|
| 276 |
+ assert os.path.exists(source)
|
|
| 277 |
+ |
|
| 278 |
+ setup_test()
|
|
| 279 |
+ result = cli.run(project=project_path, silent=True,
|
|
| 280 |
+ args=['show', "element{}.bst".format(str(dependency_depth))])
|
|
| 281 |
+ |
|
| 282 |
+ recursion_limit = sys.getrecursionlimit()
|
|
| 283 |
+ if dependency_depth <= recursion_limit:
|
|
| 284 |
+ result.assert_success()
|
|
| 285 |
+ else:
|
|
| 286 |
+ # Assert exception is thown and handled
|
|
| 287 |
+ assert not result.unhandled_exception
|
|
| 288 |
+ assert result.exit_code == -1
|
|
| 289 |
+ |
|
| 290 |
+ shutil.rmtree(project_path)
|
| ... | ... | @@ -123,6 +123,58 @@ def test_open_force(cli, tmpdir, datafiles, kind): |
| 123 | 123 |
result.assert_success()
|
| 124 | 124 |
|
| 125 | 125 |
|
| 126 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 127 |
+@pytest.mark.parametrize("kind", repo_kinds)
|
|
| 128 |
+def test_open_force_open(cli, tmpdir, datafiles, kind):
|
|
| 129 |
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False)
|
|
| 130 |
+ |
|
| 131 |
+ # Assert the workspace dir exists
|
|
| 132 |
+ assert os.path.exists(workspace)
|
|
| 133 |
+ |
|
| 134 |
+ # Now open the workspace again with --force, this should happily succeed
|
|
| 135 |
+ result = cli.run(project=project, args=[
|
|
| 136 |
+ 'workspace', 'open', '--force', element_name, workspace
|
|
| 137 |
+ ])
|
|
| 138 |
+ result.assert_success()
|
|
| 139 |
+ |
|
| 140 |
+ |
|
| 141 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 142 |
+@pytest.mark.parametrize("kind", repo_kinds)
|
|
| 143 |
+def test_open_force_different_workspace(cli, tmpdir, datafiles, kind):
|
|
| 144 |
+ element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False, "-alpha")
|
|
| 145 |
+ |
|
| 146 |
+ # Assert the workspace dir exists
|
|
| 147 |
+ assert os.path.exists(workspace)
|
|
| 148 |
+ |
|
| 149 |
+ hello_path = os.path.join(workspace, 'usr', 'bin', 'hello')
|
|
| 150 |
+ hello1_path = os.path.join(workspace, 'usr', 'bin', 'hello1')
|
|
| 151 |
+ |
|
| 152 |
+ tmpdir = os.path.join(str(tmpdir), "-beta")
|
|
| 153 |
+ shutil.move(hello_path, hello1_path)
|
|
| 154 |
+ element_name2, project2, workspace2 = open_workspace(cli, tmpdir, datafiles, kind, False, "-beta")
|
|
| 155 |
+ |
|
| 156 |
+ # Assert the workspace dir exists
|
|
| 157 |
+ assert os.path.exists(workspace2)
|
|
| 158 |
+ |
|
| 159 |
+ # Assert that workspace 1 contains the modified file
|
|
| 160 |
+ assert os.path.exists(hello1_path)
|
|
| 161 |
+ |
|
| 162 |
+ # Assert that workspace 2 contains the unmodified file
|
|
| 163 |
+ assert os.path.exists(os.path.join(workspace2, 'usr', 'bin', 'hello'))
|
|
| 164 |
+ |
|
| 165 |
+ # Now open the workspace again with --force, this should happily succeed
|
|
| 166 |
+ result = cli.run(project=project, args=[
|
|
| 167 |
+ 'workspace', 'open', '--force', element_name2, workspace
|
|
| 168 |
+ ])
|
|
| 169 |
+ |
|
| 170 |
+ # Assert that the file in workspace 1 has been replaced
|
|
| 171 |
+ # With the file from workspace 2
|
|
| 172 |
+ assert os.path.exists(hello_path)
|
|
| 173 |
+ assert not os.path.exists(hello1_path)
|
|
| 174 |
+ |
|
| 175 |
+ result.assert_success()
|
|
| 176 |
+ |
|
| 177 |
+ |
|
| 126 | 178 |
@pytest.mark.datafiles(DATA_DIR)
|
| 127 | 179 |
@pytest.mark.parametrize("kind", repo_kinds)
|
| 128 | 180 |
def test_close(cli, tmpdir, datafiles, kind):
|
| 1 |
+import os
|
|
| 2 |
+import pytest
|
|
| 3 |
+ |
|
| 4 |
+from buildstream._exceptions import ErrorDomain
|
|
| 5 |
+from buildstream import _yaml
|
|
| 6 |
+from tests.testutils import cli
|
|
| 7 |
+ |
|
| 8 |
+DATA_DIR = os.path.join(
|
|
| 9 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
| 10 |
+ 'remote',
|
|
| 11 |
+)
|
|
| 12 |
+ |
|
| 13 |
+ |
|
| 14 |
+def generate_project(project_dir, tmpdir):
|
|
| 15 |
+ project_file = os.path.join(project_dir, "project.conf")
|
|
| 16 |
+ _yaml.dump({
|
|
| 17 |
+ 'name': 'foo',
|
|
| 18 |
+ 'aliases': {
|
|
| 19 |
+ 'tmpdir': "file:///" + str(tmpdir)
|
|
| 20 |
+ }
|
|
| 21 |
+ }, project_file)
|
|
| 22 |
+ |
|
| 23 |
+ |
|
| 24 |
+# Test that without ref, consistency is set appropriately.
|
|
| 25 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
|
|
| 26 |
+def test_no_ref(cli, tmpdir, datafiles):
|
|
| 27 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 28 |
+ generate_project(project, tmpdir)
|
|
| 29 |
+ assert cli.get_element_state(project, 'target.bst') == 'no reference'
|
|
| 30 |
+ |
|
| 31 |
+ |
|
| 32 |
+# Here we are doing a fetch on a file that doesn't exist. target.bst
|
|
| 33 |
+# refers to 'file' but that file is not present.
|
|
| 34 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'missing-file'))
|
|
| 35 |
+def test_missing_file(cli, tmpdir, datafiles):
|
|
| 36 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 37 |
+ generate_project(project, tmpdir)
|
|
| 38 |
+ |
|
| 39 |
+ # Try to fetch it
|
|
| 40 |
+ result = cli.run(project=project, args=[
|
|
| 41 |
+ 'fetch', 'target.bst'
|
|
| 42 |
+ ])
|
|
| 43 |
+ |
|
| 44 |
+ result.assert_main_error(ErrorDomain.STREAM, None)
|
|
| 45 |
+ result.assert_task_error(ErrorDomain.SOURCE, None)
|
|
| 46 |
+ |
|
| 47 |
+ |
|
| 48 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'path-in-filename'))
|
|
| 49 |
+def test_path_in_filename(cli, tmpdir, datafiles):
|
|
| 50 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 51 |
+ generate_project(project, tmpdir)
|
|
| 52 |
+ |
|
| 53 |
+ # Try to fetch it
|
|
| 54 |
+ result = cli.run(project=project, args=[
|
|
| 55 |
+ 'fetch', 'target.bst'
|
|
| 56 |
+ ])
|
|
| 57 |
+ |
|
| 58 |
+ # The bst file has a / in the filename param
|
|
| 59 |
+ result.assert_main_error(ErrorDomain.SOURCE, None)
|
|
| 60 |
+ |
|
| 61 |
+ |
|
| 62 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file'))
|
|
| 63 |
+def test_simple_file_build(cli, tmpdir, datafiles):
|
|
| 64 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 65 |
+ generate_project(project, tmpdir)
|
|
| 66 |
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
|
|
| 67 |
+ |
|
| 68 |
+ # Try to fetch it
|
|
| 69 |
+ result = cli.run(project=project, args=[
|
|
| 70 |
+ 'fetch', 'target.bst'
|
|
| 71 |
+ ])
|
|
| 72 |
+ result.assert_success()
|
|
| 73 |
+ |
|
| 74 |
+ result = cli.run(project=project, args=[
|
|
| 75 |
+ 'build', 'target.bst'
|
|
| 76 |
+ ])
|
|
| 77 |
+ result.assert_success()
|
|
| 78 |
+ |
|
| 79 |
+ result = cli.run(project=project, args=[
|
|
| 80 |
+ 'checkout', 'target.bst', checkoutdir
|
|
| 81 |
+ ])
|
|
| 82 |
+ result.assert_success()
|
|
| 83 |
+ # Note that the url of the file in target.bst is actually /dir/file
|
|
| 84 |
+ # but this tests confirms we take the basename
|
|
| 85 |
+ assert(os.path.exists(os.path.join(checkoutdir, 'file')))
|
|
| 86 |
+ |
|
| 87 |
+ |
|
| 88 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
|
|
| 89 |
+def test_simple_file_custom_name_build(cli, tmpdir, datafiles):
|
|
| 90 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 91 |
+ generate_project(project, tmpdir)
|
|
| 92 |
+ checkoutdir = os.path.join(str(tmpdir), "checkout")
|
|
| 93 |
+ |
|
| 94 |
+ # Try to fetch it
|
|
| 95 |
+ result = cli.run(project=project, args=[
|
|
| 96 |
+ 'fetch', 'target.bst'
|
|
| 97 |
+ ])
|
|
| 98 |
+ result.assert_success()
|
|
| 99 |
+ |
|
| 100 |
+ result = cli.run(project=project, args=[
|
|
| 101 |
+ 'build', 'target.bst'
|
|
| 102 |
+ ])
|
|
| 103 |
+ result.assert_success()
|
|
| 104 |
+ |
|
| 105 |
+ result = cli.run(project=project, args=[
|
|
| 106 |
+ 'checkout', 'target.bst', checkoutdir
|
|
| 107 |
+ ])
|
|
| 108 |
+ result.assert_success()
|
|
| 109 |
+ assert(not os.path.exists(os.path.join(checkoutdir, 'file')))
|
|
| 110 |
+ assert(os.path.exists(os.path.join(checkoutdir, 'custom-file')))
|
|
| 111 |
+ |
|
| 112 |
+ |
|
| 113 |
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
|
|
| 114 |
+def test_unique_key(cli, tmpdir, datafiles):
|
|
| 115 |
+ '''This test confirms that the 'filename' parameter is honoured when it comes
|
|
| 116 |
+ to generating a cache key for the source.
|
|
| 117 |
+ '''
|
|
| 118 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 119 |
+ generate_project(project, tmpdir)
|
|
| 120 |
+ assert cli.get_element_state(project, 'target.bst') == "fetch needed"
|
|
| 121 |
+ assert cli.get_element_state(project, 'target-custom.bst') == "fetch needed"
|
|
| 122 |
+ # Try to fetch it
|
|
| 123 |
+ result = cli.run(project=project, args=[
|
|
| 124 |
+ 'fetch', 'target.bst'
|
|
| 125 |
+ ])
|
|
| 126 |
+ |
|
| 127 |
+ # We should download the file only once
|
|
| 128 |
+ assert cli.get_element_state(project, 'target.bst') == 'buildable'
|
|
| 129 |
+ assert cli.get_element_state(project, 'target-custom.bst') == 'buildable'
|
|
| 130 |
+ |
|
| 131 |
+ # But the cache key is different because the 'filename' is different.
|
|
| 132 |
+ assert cli.get_element_key(project, 'target.bst') != \
|
|
| 133 |
+ cli.get_element_key(project, 'target-custom.bst')
|
| 1 |
+kind: autotools
|
|
| 2 |
+description: The kind of this element is irrelevant.
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/file
|
|
| 6 |
+ ref: abcdef
|
|
| 7 |
+ filename: filename
|
| 1 |
+filecontent
|
| 1 |
+kind: autotools
|
|
| 2 |
+description: The kind of this element is irrelevant.
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/file
|
| 1 |
+kind: import
|
|
| 2 |
+description: test
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/dir/file
|
|
| 6 |
+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
| 7 |
+ filename: path/to/file
|
| 1 |
+kind: import
|
|
| 2 |
+description: test
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/dir/file
|
|
| 6 |
+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
| 7 |
+ filename: custom-file
|
| 1 |
+kind: import
|
|
| 2 |
+description: test
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/dir/file
|
|
| 6 |
+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
| 1 |
+kind: import
|
|
| 2 |
+description: test
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/dir/file
|
|
| 6 |
+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
|
| 7 |
+ filename: some-custom-file
|
| 1 |
+kind: import
|
|
| 2 |
+description: test
|
|
| 3 |
+sources:
|
|
| 4 |
+- kind: remote
|
|
| 5 |
+ url: tmpdir:/dir/file
|
|
| 6 |
+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
|
