[Notes] [Git][BuildStream/buildstream][tristan/reduce-gitlab-ci] 5 commits: Remove artifact extracts when artifact expires in cache



Title: GitLab

Tristan Van Berkom pushed to branch tristan/reduce-gitlab-ci at BuildStream / buildstream

Commits:

5 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -10,6 +10,16 @@ stages:
    10 10
       - test
    
    11 11
       - post
    
    12 12
     
    
    13
    +# Avoid running all the tests post merge on
    
    14
    +# master or on any release branch.
    
    15
    +#
    
    16
    +.tests-condition-template: &tests-condition
    
    17
    +  only:
    
    18
    +  - branches
    
    19
    +  except:
    
    20
    +  - master
    
    21
    +  - /bst-1\..*/
    
    22
    +
    
    13 23
     #####################################################
    
    14 24
     #                  Prepare stage                    #
    
    15 25
     #####################################################
    
    ... ... @@ -81,20 +91,26 @@ source_dist:
    81 91
     tests-debian-9:
    
    82 92
       image: buildstream/testsuite-debian:9-master-114-4cab18e3
    
    83 93
       <<: *linux-tests
    
    94
    +  <<: *tests-condition
    
    84 95
     
    
    85 96
     tests-fedora-27:
    
    86 97
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    87 98
       <<: *linux-tests
    
    99
    +  <<: *tests-condition
    
    88 100
     
    
    89 101
     tests-fedora-28:
    
    90 102
       image: buildstream/testsuite-fedora:28-master-114-4cab18e3
    
    91 103
       <<: *linux-tests
    
    104
    +  <<: *tests-condition
    
    92 105
     
    
    93 106
     tests-ubuntu-18.04:
    
    94 107
       image: buildstream/testsuite-ubuntu:18.04-master-114-4cab18e3
    
    95 108
       <<: *linux-tests
    
    109
    +  <<: *tests-condition
    
    96 110
     
    
    97 111
     tests-unix:
    
    112
    +  <<: *tests-condition
    
    113
    +
    
    98 114
       # Use fedora here, to a) run a test on fedora and b) ensure that we
    
    99 115
       # can get rid of ostree - this is not possible with debian-8
    
    100 116
       image: buildstream/testsuite-fedora:27-master-114-4cab18e3
    
    ... ... @@ -133,6 +149,15 @@ tests-unix:
    133 149
     # Note: We still do not enforce a consistent installation of python3-sphinx,
    
    134 150
     #       as it will significantly grow the backing image.
    
    135 151
     docs:
    
    152
    +
    
    153
    +  # Here we build the docs for every pre-merge CI, but avoid
    
    154
    +  # the job on post-merge to stable branches, because we only
    
    155
    +  # ever publish them from master
    
    156
    +  only:
    
    157
    +  - branches
    
    158
    +  except:
    
    159
    +  - /bst-1\..*/
    
    160
    +
    
    136 161
       stage: test
    
    137 162
       script:
    
    138 163
       - export BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources"
    
    ... ... @@ -157,6 +182,8 @@ docs:
    157 182
     # as an output of radon, with some conversion
    
    158 183
     #
    
    159 184
     codequality:
    
    185
    +  <<: *tests-condition
    
    186
    +
    
    160 187
       image: docker:stable
    
    161 188
       stage: post
    
    162 189
       variables:
    
    ... ... @@ -175,6 +202,8 @@ codequality:
    175 202
         paths: [codeclimate.json]
    
    176 203
     
    
    177 204
     analysis:
    
    205
    +  <<: *tests-condition
    
    206
    +
    
    178 207
       stage: post
    
    179 208
       script:
    
    180 209
       - |
    
    ... ... @@ -203,6 +232,8 @@ analysis:
    203 232
     # Collate coverage reports
    
    204 233
     #
    
    205 234
     coverage:
    
    235
    +  <<: *tests-condition
    
    236
    +
    
    206 237
       stage: post
    
    207 238
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    
    208 239
       script:
    

  • README.rst
    1 1
     About
    
    2 2
     -----
    
    3
    -.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/pipeline.svg
    
    4
    -   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    5
    -
    
    6
    -.. image:: https://gitlab.com/BuildStream/buildstream/badges/master/coverage.svg?job=coverage
    
    7
    -   :target: https://gitlab.com/BuildStream/buildstream/commits/master
    
    8 3
     
    
    9 4
     
    
    10 5
     What is BuildStream?
    

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -30,6 +30,8 @@ from urllib.parse import urlparse
    30 30
     
    
    31 31
     import grpc
    
    32 32
     
    
    33
    +from .. import _yaml
    
    34
    +
    
    33 35
     from .._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
    
    34 36
     from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    
    35 37
     from .._protos.buildstream.v2 import buildstream_pb2, buildstream_pb2_grpc
    
    ... ... @@ -526,6 +528,25 @@ class CASCache(ArtifactCache):
    526 528
         #
    
    527 529
         def remove(self, ref, *, defer_prune=False):
    
    528 530
     
    
    531
    +        # Remove extract if not used by other ref
    
    532
    +        tree = self.resolve_ref(ref)
    
    533
    +        ref_name, ref_hash = os.path.split(ref)
    
    534
    +        extract = os.path.join(self.extractdir, ref_name, tree.hash)
    
    535
    +        keys_file = os.path.join(extract, 'meta', 'keys.yaml')
    
    536
    +        if os.path.exists(keys_file):
    
    537
    +            keys_meta = _yaml.load(keys_file)
    
    538
    +            keys = [keys_meta['strong'], keys_meta['weak']]
    
    539
    +            remove_extract = True
    
    540
    +            for other_hash in keys:
    
    541
    +                if other_hash == ref_hash:
    
    542
    +                    continue
    
    543
    +                remove_extract = False
    
    544
    +                break
    
    545
    +
    
    546
    +            if remove_extract:
    
    547
    +                utils._force_rmtree(extract)
    
    548
    +
    
    549
    +        # Remove cache ref
    
    529 550
             refpath = self._refpath(ref)
    
    530 551
             if not os.path.exists(refpath):
    
    531 552
                 raise ArtifactError("Could not find artifact for ref '{}'".format(ref))
    

  • setup.py
    ... ... @@ -273,7 +273,7 @@ setup(name='BuildStream',
    273 273
               'ruamel.yaml < 0.15.52',
    
    274 274
               'pluginbase',
    
    275 275
               'Click',
    
    276
    -          'blessings',
    
    276
    +          'blessings >= 1.6',
    
    277 277
               'jinja2 >= 2.10',
    
    278 278
               'protobuf >= 3.5',
    
    279 279
               'grpcio >= 1.10',
    

  • tests/artifactcache/expiry.py
    ... ... @@ -268,3 +268,38 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
    268 268
             res.assert_success()
    
    269 269
         else:
    
    270 270
             res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
    
    271
    +
    
    272
    +
    
    273
    +@pytest.mark.datafiles(DATA_DIR)
    
    274
    +def test_extract_expiry(cli, datafiles, tmpdir):
    
    275
    +    project = os.path.join(datafiles.dirname, datafiles.basename)
    
    276
    +    element_path = 'elements'
    
    277
    +
    
    278
    +    cli.configure({
    
    279
    +        'cache': {
    
    280
    +            'quota': 10000000,
    
    281
    +        }
    
    282
    +    })
    
    283
    +
    
    284
    +    create_element_size('target.bst', project, element_path, [], 6000000)
    
    285
    +    res = cli.run(project=project, args=['build', 'target.bst'])
    
    286
    +    res.assert_success()
    
    287
    +    assert cli.get_element_state(project, 'target.bst') == 'cached'
    
    288
    +
    
    289
    +    # Force creating extract
    
    290
    +    res = cli.run(project=project, args=['checkout', 'target.bst', os.path.join(str(tmpdir), 'checkout')])
    
    291
    +    res.assert_success()
    
    292
    +
    
    293
    +    extractdir = os.path.join(project, 'cache', 'artifacts', 'extract', 'test', 'target')
    
    294
    +    extracts = os.listdir(extractdir)
    
    295
    +    assert(len(extracts) == 1)
    
    296
    +    extract = os.path.join(extractdir, extracts[0])
    
    297
    +
    
    298
    +    # Remove target.bst from artifact cache
    
    299
    +    create_element_size('target2.bst', project, element_path, [], 6000000)
    
    300
    +    res = cli.run(project=project, args=['build', 'target2.bst'])
    
    301
    +    res.assert_success()
    
    302
    +    assert cli.get_element_state(project, 'target.bst') != 'cached'
    
    303
    +
    
    304
    +    # Now the extract should be removed.
    
    305
    +    assert not os.path.exists(extract)



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]