Jürg Billeter pushed to branch master at BuildStream / buildstream
Commits:
-
2b767fe8
by Jürg Billeter at 2018-12-20T10:06:11Z
-
7a102144
by Jürg Billeter at 2018-12-20T10:06:11Z
-
b325989e
by Jürg Billeter at 2018-12-20T10:07:20Z
-
77d8ad45
by Jürg Billeter at 2018-12-20T10:42:39Z
6 changed files:
- buildstream/_scheduler/queues/fetchqueue.py
- buildstream/element.py
- + tests/sources/no-fetch-cached/files/file
- + tests/sources/no-fetch-cached/plugins/sources/always_cached.py
- + tests/sources/no-fetch-cached/project.conf
- + tests/sources/no_fetch_cached.py
Changes:
| ... | ... | @@ -40,10 +40,7 @@ class FetchQueue(Queue): |
| 40 | 40 |
self._skip_cached = skip_cached
|
| 41 | 41 |
|
| 42 | 42 |
def process(self, element):
|
| 43 |
- previous_sources = []
|
|
| 44 |
- for source in element.sources():
|
|
| 45 |
- source._fetch(previous_sources)
|
|
| 46 |
- previous_sources.append(source)
|
|
| 43 |
+ element._fetch()
|
|
| 47 | 44 |
|
| 48 | 45 |
def status(self, element):
|
| 49 | 46 |
# state of dependencies may have changed, recalculate element state
|
| ... | ... | @@ -2022,6 +2022,20 @@ class Element(Plugin): |
| 2022 | 2022 |
|
| 2023 | 2023 |
return True
|
| 2024 | 2024 |
|
| 2025 |
+ # _fetch()
|
|
| 2026 |
+ #
|
|
| 2027 |
+ # Fetch the element's sources.
|
|
| 2028 |
+ #
|
|
| 2029 |
+ # Raises:
|
|
| 2030 |
+ # SourceError: If one of the element sources has an error
|
|
| 2031 |
+ #
|
|
| 2032 |
+ def _fetch(self):
|
|
| 2033 |
+ previous_sources = []
|
|
| 2034 |
+ for source in self.sources():
|
|
| 2035 |
+ if source._get_consistency() < Consistency.CACHED:
|
|
| 2036 |
+ source._fetch(previous_sources)
|
|
| 2037 |
+ previous_sources.append(source)
|
|
| 2038 |
+ |
|
| 2025 | 2039 |
#############################################################
|
| 2026 | 2040 |
# Private Local Methods #
|
| 2027 | 2041 |
#############################################################
|
| 1 |
+Hello World!
|
| 1 |
+"""
|
|
| 2 |
+always_cached
|
|
| 3 |
+=============
|
|
| 4 |
+ |
|
| 5 |
+This is a test source plugin that is always cached.
|
|
| 6 |
+Used to test that BuildStream core does not call fetch() for cached sources.
|
|
| 7 |
+ |
|
| 8 |
+"""
|
|
| 9 |
+ |
|
| 10 |
+from buildstream import Consistency, Source
|
|
| 11 |
+ |
|
| 12 |
+ |
|
| 13 |
+class AlwaysCachedSource(Source):
|
|
| 14 |
+ |
|
| 15 |
+ def configure(self, node):
|
|
| 16 |
+ pass
|
|
| 17 |
+ |
|
| 18 |
+ def preflight(self):
|
|
| 19 |
+ pass
|
|
| 20 |
+ |
|
| 21 |
+ def get_unique_key(self):
|
|
| 22 |
+ return None
|
|
| 23 |
+ |
|
| 24 |
+ def get_consistency(self):
|
|
| 25 |
+ return Consistency.CACHED
|
|
| 26 |
+ |
|
| 27 |
+ def load_ref(self, node):
|
|
| 28 |
+ pass
|
|
| 29 |
+ |
|
| 30 |
+ def get_ref(self):
|
|
| 31 |
+ return None
|
|
| 32 |
+ |
|
| 33 |
+ def set_ref(self, ref, node):
|
|
| 34 |
+ pass
|
|
| 35 |
+ |
|
| 36 |
+ def fetch(self):
|
|
| 37 |
+ # Source is always cached, so fetch() should never be called
|
|
| 38 |
+ assert False
|
|
| 39 |
+ |
|
| 40 |
+ def stage(self, directory):
|
|
| 41 |
+ pass
|
|
| 42 |
+ |
|
| 43 |
+ |
|
| 44 |
+def setup():
|
|
| 45 |
+ return AlwaysCachedSource
|
| 1 |
+# Project with local source plugins
|
|
| 2 |
+name: no-fetch-cached
|
|
| 3 |
+ |
|
| 4 |
+plugins:
|
|
| 5 |
+- origin: local
|
|
| 6 |
+ path: plugins/sources
|
|
| 7 |
+ sources:
|
|
| 8 |
+ always_cached: 0
|
| 1 |
+import os
|
|
| 2 |
+import pytest
|
|
| 3 |
+ |
|
| 4 |
+from buildstream import _yaml
|
|
| 5 |
+ |
|
| 6 |
+from tests.testutils import cli, create_repo
|
|
| 7 |
+from tests.testutils.site import HAVE_GIT
|
|
| 8 |
+ |
|
| 9 |
+DATA_DIR = os.path.join(
|
|
| 10 |
+ os.path.dirname(os.path.realpath(__file__)),
|
|
| 11 |
+ 'no-fetch-cached'
|
|
| 12 |
+)
|
|
| 13 |
+ |
|
| 14 |
+ |
|
| 15 |
+##################################################################
|
|
| 16 |
+# Tests #
|
|
| 17 |
+##################################################################
|
|
| 18 |
+# Test that fetch() is not called for cached sources
|
|
| 19 |
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
|
|
| 20 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
| 21 |
+def test_no_fetch_cached(cli, tmpdir, datafiles):
|
|
| 22 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
| 23 |
+ |
|
| 24 |
+ # Create the repo from 'files' subdir
|
|
| 25 |
+ repo = create_repo('git', str(tmpdir))
|
|
| 26 |
+ ref = repo.create(os.path.join(project, 'files'))
|
|
| 27 |
+ |
|
| 28 |
+ # Write out test target with a cached and a non-cached source
|
|
| 29 |
+ element = {
|
|
| 30 |
+ 'kind': 'import',
|
|
| 31 |
+ 'sources': [
|
|
| 32 |
+ repo.source_config(ref=ref),
|
|
| 33 |
+ {
|
|
| 34 |
+ 'kind': 'always_cached'
|
|
| 35 |
+ }
|
|
| 36 |
+ ]
|
|
| 37 |
+ }
|
|
| 38 |
+ _yaml.dump(element, os.path.join(project, 'target.bst'))
|
|
| 39 |
+ |
|
| 40 |
+ # Test fetch of target with a cached and a non-cached source
|
|
| 41 |
+ result = cli.run(project=project, args=[
|
|
| 42 |
+ 'source', 'fetch', 'target.bst'
|
|
| 43 |
+ ])
|
|
| 44 |
+ result.assert_success()
|