Tristan Van Berkom pushed to branch tristan/reduce-gitlab-ci at BuildStream / buildstream
Commits:
-
2ac654a8
by Valentin David at 2018-08-23T08:02:50Z
-
176aa09f
by Tristan Van Berkom at 2018-08-23T09:10:51Z
-
f1e3f10f
by Mathieu Bridon at 2018-08-23T09:11:47Z
-
bea01531
by Tristan Van Berkom at 2018-08-23T09:14:40Z
-
da19bcf1
by Tristan Van Berkom at 2018-08-23T09:14:40Z
5 changed files:
- .gitlab-ci.yml
- README.rst
- buildstream/_artifactcache/cascache.py
- setup.py
- tests/artifactcache/expiry.py
Changes:
... | ... | @@ -10,6 +10,16 @@ stages: |
10 | 10 |
- test
|
11 | 11 |
- post
|
12 | 12 |
|
13 |
+# Avoid running all the tests post merge on
|
|
14 |
+# master or on any release branch.
|
|
15 |
+#
|
|
16 |
+.tests-condition-template: &tests-condition
|
|
17 |
+ only:
|
|
18 |
+ - branches
|
|
19 |
+ except:
|
|
20 |
+ - master
|
|
21 |
+ - /bst-1\..*/
|
|
22 |
+ |
|
13 | 23 |
#####################################################
|
14 | 24 |
# Prepare stage #
|
15 | 25 |
#####################################################
|
... | ... | @@ -81,20 +91,26 @@ source_dist: |
81 | 91 |
tests-debian-9:
|
82 | 92 |
image: buildstream/testsuite-debian:9-master-114-4cab18e3
|
83 | 93 |
<<: *linux-tests
|
94 |
+ <<: *tests-condition
|
|
84 | 95 |
|
85 | 96 |
tests-fedora-27:
|
86 | 97 |
image: buildstream/testsuite-fedora:27-master-114-4cab18e3
|
87 | 98 |
<<: *linux-tests
|
99 |
+ <<: *tests-condition
|
|
88 | 100 |
|
89 | 101 |
tests-fedora-28:
|
90 | 102 |
image: buildstream/testsuite-fedora:28-master-114-4cab18e3
|
91 | 103 |
<<: *linux-tests
|
104 |
+ <<: *tests-condition
|
|
92 | 105 |
|
93 | 106 |
tests-ubuntu-18.04:
|
94 | 107 |
image: buildstream/testsuite-ubuntu:18.04-master-114-4cab18e3
|
95 | 108 |
<<: *linux-tests
|
109 |
+ <<: *tests-condition
|
|
96 | 110 |
|
97 | 111 |
tests-unix:
|
112 |
+ <<: *tests-condition
|
|
113 |
+ |
|
98 | 114 |
# Use fedora here, to a) run a test on fedora and b) ensure that we
|
99 | 115 |
# can get rid of ostree - this is not possible with debian-8
|
100 | 116 |
image: buildstream/testsuite-fedora:27-master-114-4cab18e3
|
... | ... | @@ -133,6 +149,15 @@ tests-unix: |
133 | 149 |
# Note: We still do not enforce a consistent installation of python3-sphinx,
|
134 | 150 |
# as it will significantly grow the backing image.
|
135 | 151 |
docs:
|
152 |
+ |
|
153 |
+ # Here we build the docs for every pre-merge CI, but avoid
|
|
154 |
+ # the job on post-merge to stable branches, because we only
|
|
155 |
+ # ever publish them from master
|
|
156 |
+ only:
|
|
157 |
+ - branches
|
|
158 |
+ except:
|
|
159 |
+ - /bst-1\..*/
|
|
160 |
+ |
|
136 | 161 |
stage: test
|
137 | 162 |
script:
|
138 | 163 |
- export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
|
... | ... | @@ -157,6 +182,8 @@ docs: |
157 | 182 |
# as an output of radon, with some conversion
|
158 | 183 |
#
|
159 | 184 |
codequality:
|
185 |
+ <<: *tests-condition
|
|
186 |
+ |
|
160 | 187 |
image: docker:stable
|
161 | 188 |
stage: post
|
162 | 189 |
variables:
|
... | ... | @@ -175,6 +202,8 @@ codequality: |
175 | 202 |
paths: [codeclimate.json]
|
176 | 203 |
|
177 | 204 |
analysis:
|
205 |
+ <<: *tests-condition
|
|
206 |
+ |
|
178 | 207 |
stage: post
|
179 | 208 |
script:
|
180 | 209 |
- |
|
... | ... | @@ -203,6 +232,8 @@ analysis: |
203 | 232 |
# Collate coverage reports
|
204 | 233 |
#
|
205 | 234 |
coverage:
|
235 |
+ <<: *tests-condition
|
|
236 |
+ |
|
206 | 237 |
stage: post
|
207 | 238 |
coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
|
208 | 239 |
script:
|
1 | 1 |
About
|
2 | 2 |
-----
|
3 |
-.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/pipeline.svg
|
|
4 |
- :target: https://gitlab.com/BuildStream/buildstream/commits/master
|
|
5 |
- |
|
6 |
-.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
|
|
7 |
- :target: https://gitlab.com/BuildStream/buildstream/commits/master
|
|
8 | 3 |
|
9 | 4 |
|
10 | 5 |
What is BuildStream?
|
... | ... | @@ -30,6 +30,8 @@ from urllib.parse import urlparse |
30 | 30 |
|
31 | 31 |
import grpc
|
32 | 32 |
|
33 |
+from .. import _yaml
|
|
34 |
+ |
|
33 | 35 |
from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
|
34 | 36 |
from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
|
35 | 37 |
from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
|
... | ... | @@ -526,6 +528,25 @@ class CASCache(ArtifactCache): |
526 | 528 |
#
|
527 | 529 |
def remove(self, ref, *, defer_prune=False):
|
528 | 530 |
|
531 |
+ # Remove extract if not used by other ref
|
|
532 |
+ tree = self.resolve_ref(ref)
|
|
533 |
+ ref_name, ref_hash = os.path.split(ref)
|
|
534 |
+ extract = os.path.join(self.extractdir, ref_name, tree.hash)
|
|
535 |
+ keys_file = os.path.join(extract, 'meta', 'keys.yaml')
|
|
536 |
+ if os.path.exists(keys_file):
|
|
537 |
+ keys_meta = _yaml.load(keys_file)
|
|
538 |
+ keys = [keys_meta['strong'], keys_meta['weak']]
|
|
539 |
+ remove_extract = True
|
|
540 |
+ for other_hash in keys:
|
|
541 |
+ if other_hash == ref_hash:
|
|
542 |
+ continue
|
|
543 |
+ remove_extract = False
|
|
544 |
+ break
|
|
545 |
+ |
|
546 |
+ if remove_extract:
|
|
547 |
+ utils._force_rmtree(extract)
|
|
548 |
+ |
|
549 |
+ # Remove cache ref
|
|
529 | 550 |
refpath = self._refpath(ref)
|
530 | 551 |
if not os.path.exists(refpath):
|
531 | 552 |
raise ArtifactError("Could not find artifact for ref '{}'".format(ref))
|
... | ... | @@ -273,7 +273,7 @@ setup(name='BuildStream', |
273 | 273 |
'ruamel.yaml < 0.15.52',
|
274 | 274 |
'pluginbase',
|
275 | 275 |
'Click',
|
276 |
- 'blessings',
|
|
276 |
+ 'blessings >= 1.6',
|
|
277 | 277 |
'jinja2 >= 2.10',
|
278 | 278 |
'protobuf >= 3.5',
|
279 | 279 |
'grpcio >= 1.10',
|
... | ... | @@ -268,3 +268,38 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success): |
268 | 268 |
res.assert_success()
|
269 | 269 |
else:
|
270 | 270 |
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
271 |
+ |
|
272 |
+ |
|
273 |
+@pytest.mark.datafiles(DATA_DIR)
|
|
274 |
+def test_extract_expiry(cli, datafiles, tmpdir):
|
|
275 |
+ project = os.path.join(datafiles.dirname, datafiles.basename)
|
|
276 |
+ element_path = 'elements'
|
|
277 |
+ |
|
278 |
+ cli.configure({
|
|
279 |
+ 'cache': {
|
|
280 |
+ 'quota': 10000000,
|
|
281 |
+ }
|
|
282 |
+ })
|
|
283 |
+ |
|
284 |
+ create_element_size('target.bst', project, element_path, [], 6000000)
|
|
285 |
+ res = cli.run(project=project, args=['build', 'target.bst'])
|
|
286 |
+ res.assert_success()
|
|
287 |
+ assert cli.get_element_state(project, 'target.bst') == 'cached'
|
|
288 |
+ |
|
289 |
+ # Force creating extract
|
|
290 |
+ res = cli.run(project=project, args=['checkout', 'target.bst', os.path.join(str(tmpdir), 'checkout')])
|
|
291 |
+ res.assert_success()
|
|
292 |
+ |
|
293 |
+ extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
|
|
294 |
+ extracts = os.listdir(extractdir)
|
|
295 |
+ assert(len(extracts) == 1)
|
|
296 |
+ extract = os.path.join(extractdir, extracts[0])
|
|
297 |
+ |
|
298 |
+ # Remove target.bst from artifact cache
|
|
299 |
+ create_element_size('target2.bst', project, element_path, [], 6000000)
|
|
300 |
+ res = cli.run(project=project, args=['build', 'target2.bst'])
|
|
301 |
+ res.assert_success()
|
|
302 |
+ assert cli.get_element_state(project, 'target.bst') != 'cached'
|
|
303 |
+ |
|
304 |
+ # Now the extract should be removed.
|
|
305 |
+ assert not os.path.exists(extract)
|