[Notes] [Git][BuildStream/buildstream][raoul/870-root-cache-dir] rootcachedir: new dir option that's base to other dirs



Title: GitLab

Raoul Hidalgo Charman pushed to branch raoul/870-root-cache-dir at BuildStream / buildstream

Commits:

17 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -164,6 +164,7 @@ docs:
    164 164
       variables:
    
    165 165
         BST_FORCE_SESSION_REBUILD: 1
    
    166 166
       script:
    
    167
    +  - tree "$(pwd)"
    
    167 168
       - env BST_SOURCE_CACHE="$(pwd)/cache/integration-cache/sources" tox -e docs
    
    168 169
       - mv doc/build/html public
    
    169 170
       except:
    

  • buildstream/_context.py
    ... ... @@ -58,12 +58,21 @@ class Context():
    58 58
             # Filename indicating which configuration file was used, or None for the defaults
    
    59 59
             self.config_origin = None
    
    60 60
     
    
    61
    +        # The directory under which other directories are based
    
    62
    +        self.rootcachedir = None
    
    63
    +
    
    61 64
             # The directory where various sources are stored
    
    62 65
             self.sourcedir = None
    
    63 66
     
    
    64 67
             # The directory where build sandboxes will be created
    
    65 68
             self.builddir = None
    
    66 69
     
    
    70
    +        # The directory for CAS
    
    71
    +        self.casdir = None
    
    72
    +
    
    73
    +        # The directory for temporary files
    
    74
    +        self.tmpdir = None
    
    75
    +
    
    67 76
             # Default root location for workspaces
    
    68 77
             self.workspacedir = None
    
    69 78
     
    
    ... ... @@ -188,13 +197,30 @@ class Context():
    188 197
                 user_config = _yaml.load(config)
    
    189 198
                 _yaml.composite(defaults, user_config)
    
    190 199
     
    
    200
    +        # Give deprecation warnings
    
    201
    +        if defaults.get('builddir'):
    
    202
    +            print("builddir is deprecated, use rootcachedir")
    
    203
    +        else:
    
    204
    +            defaults['builddir'] = os.path.join(defaults['rootcachedir'], 'build')
    
    205
    +
    
    206
    +        if defaults.get('artifactdir'):
    
    207
    +            print("artifactdir is deprecated, use rootcachedir")
    
    208
    +        else:
    
    209
    +            defaults['artifactdir'] = os.path.join(defaults['rootcachedir'], 'artifacts')
    
    210
    +
    
    191 211
             _yaml.node_validate(defaults, [
    
    192
    -            'sourcedir', 'builddir', 'artifactdir', 'logdir',
    
    212
    +            'rootcachedir', 'sourcedir', 'builddir', 'artifactdir', 'logdir',
    
    193 213
                 'scheduler', 'artifacts', 'logging', 'projects',
    
    194
    -            'cache', 'prompt', 'workspacedir', 'remote-execution'
    
    214
    +            'cache', 'prompt', 'workspacedir', 'remote-execution',
    
    195 215
             ])
    
    196 216
     
    
    197
    -        for directory in ['sourcedir', 'builddir', 'artifactdir', 'logdir', 'workspacedir']:
    
    217
    +        # add directories not set by users
    
    218
    +        defaults['tmpdir'] = os.path.join(defaults['rootcachedir'], 'tmp')
    
    219
    +        defaults['casdir'] = os.path.join(defaults['rootcachedir'], 'cas')
    
    220
    +
    
    221
    +        for directory in ['rootcachedir', 'sourcedir', 'builddir',
    
    222
    +                          'artifactdir', 'logdir', 'workspacedir', 'casdir',
    
    223
    +                          'tmpdir']:
    
    198 224
                 # Allow the ~ tilde expansion and any environment variables in
    
    199 225
                 # path specification in the config files.
    
    200 226
                 #
    
    ... ... @@ -654,7 +680,7 @@ class Context():
    654 680
     
    
    655 681
         def get_cascache(self):
    
    656 682
             if self._cascache is None:
    
    657
    -            self._cascache = CASCache(self.artifactdir)
    
    683
    +            self._cascache = CASCache(self.rootcachedir)
    
    658 684
             return self._cascache
    
    659 685
     
    
    660 686
         # guess_element()
    

  • buildstream/data/userconfig.yaml
    ... ... @@ -13,11 +13,8 @@
    13 13
     # Location to store sources
    
    14 14
     sourcedir: ${XDG_CACHE_HOME}/buildstream/sources
    
    15 15
     
    
    16
    -# Location to perform builds
    
    17
    -builddir: ${XDG_CACHE_HOME}/buildstream/build
    
    18
    -
    
    19
    -# Location to store local binary artifacts
    
    20
    -artifactdir: ${XDG_CACHE_HOME}/buildstream/artifacts
    
    16
    +# Root location for other directories in the cache
    
    17
    +rootcachedir: ${XDG_CACHE_HOME}/buildstream
    
    21 18
     
    
    22 19
     # Location to store build logs
    
    23 20
     logdir: ${XDG_CACHE_HOME}/buildstream/logs
    

  • conftest.py
    ... ... @@ -53,15 +53,15 @@ def pytest_runtest_setup(item):
    53 53
     class IntegrationCache():
    
    54 54
     
    
    55 55
         def __init__(self, cache):
    
    56
    -        cache = os.path.abspath(cache)
    
    56
    +        self.root = os.path.abspath(cache)
    
    57 57
     
    
    58 58
             # Use the same sources every time
    
    59
    -        self.sources = os.path.join(cache, 'sources')
    
    59
    +        self.sources = os.path.join(self.root, 'sources')
    
    60 60
     
    
    61 61
             # Create a temp directory for the duration of the test for
    
    62 62
             # the artifacts directory
    
    63 63
             try:
    
    64
    -            self.artifacts = tempfile.mkdtemp(dir=cache, prefix='artifacts-')
    
    64
    +            self.artifacts = tempfile.mkdtemp(dir=self.root, prefix='artifacts-')
    
    65 65
             except OSError as e:
    
    66 66
                 raise AssertionError("Unable to create test directory !") from e
    
    67 67
     
    
    ... ... @@ -86,6 +86,10 @@ def integration_cache(request):
    86 86
             shutil.rmtree(cache.artifacts)
    
    87 87
         except FileNotFoundError:
    
    88 88
             pass
    
    89
    +    try:
    
    90
    +        shutil.rmtree(os.path.join(cache.root, 'cas'))
    
    91
    +    except FileNotFoundError:
    
    92
    +        pass
    
    89 93
     
    
    90 94
     
    
    91 95
     #################################################
    

  • doc/examples/running-commands/elements/hello.bst
    ... ... @@ -16,7 +16,8 @@ sources:
    16 16
     config:
    
    17 17
     
    
    18 18
       build-commands:
    
    19
    -  - make PREFIX="%{prefix}"
    
    19
    +  # - make PREFIX="%{prefix}"
    
    20
    +  - echo "TEST"
    
    20 21
     
    
    21
    -  install-commands:
    
    22
    -  - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install
    22
    +  # install-commands:
    
    23
    +  # - make -j1 PREFIX="%{prefix}" DESTDIR="%{install-root}" install

  • doc/sessions/running-commands.run
    ... ... @@ -2,7 +2,7 @@
    2 2
     commands:
    
    3 3
     # Make it fetch first
    
    4 4
     - directory: ../examples/running-commands
    
    5
    -  command: fetch hello.bst
    
    5
    +  command: source fetch hello.bst
    
    6 6
     
    
    7 7
     # Capture a show output
    
    8 8
     - directory: ../examples/running-commands
    

  • tests/artifactcache/expiry.py
    ... ... @@ -403,7 +403,7 @@ def test_extract_expiry(cli, datafiles, tmpdir):
    403 403
         # Now we should have a directory for the cached target2.bst, which
    
    404 404
         # replaced target.bst in the cache, we should not have a directory
    
    405 405
         # for the target.bst
    
    406
    -    refsdir = os.path.join(project, 'cache', 'artifacts', 'cas', 'refs', 'heads')
    
    406
    +    refsdir = os.path.join(project, 'cache', 'cas', 'refs', 'heads')
    
    407 407
         refsdirtest = os.path.join(refsdir, 'test')
    
    408 408
         refsdirtarget = os.path.join(refsdirtest, 'target')
    
    409 409
         refsdirtarget2 = os.path.join(refsdirtest, 'target2')
    

  • tests/artifactcache/junctions.py
    ... ... @@ -68,8 +68,8 @@ def test_push_pull(cli, tmpdir, datafiles):
    68 68
             # Now we've pushed, delete the user's local artifact cache
    
    69 69
             # directory and try to redownload it from the share
    
    70 70
             #
    
    71
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    72
    -        shutil.rmtree(artifacts)
    
    71
    +        cas = os.path.join(cli.directory, 'cas')
    
    72
    +        shutil.rmtree(cas)
    
    73 73
     
    
    74 74
             # Assert that nothing is cached locally anymore
    
    75 75
             state = cli.get_element_state(project, 'target.bst')
    

  • tests/artifactcache/pull.py
    ... ... @@ -56,7 +56,7 @@ def test_pull(cli, tmpdir, datafiles):
    56 56
         # Set up an artifact cache.
    
    57 57
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    58 58
             # Configure artifact share
    
    59
    -        artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    59
    +        cache_dir = os.path.join(str(tmpdir), 'cache')
    
    60 60
             user_config_file = str(tmpdir.join('buildstream.conf'))
    
    61 61
             user_config = {
    
    62 62
                 'scheduler': {
    
    ... ... @@ -65,7 +65,8 @@ def test_pull(cli, tmpdir, datafiles):
    65 65
                 'artifacts': {
    
    66 66
                     'url': share.repo,
    
    67 67
                     'push': True,
    
    68
    -            }
    
    68
    +            },
    
    69
    +            'rootcachedir': cache_dir
    
    69 70
             }
    
    70 71
     
    
    71 72
             # Write down the user configuration file
    
    ... ... @@ -92,7 +93,6 @@ def test_pull(cli, tmpdir, datafiles):
    92 93
             # Fake minimal context
    
    93 94
             context = Context()
    
    94 95
             context.load(config=user_config_file)
    
    95
    -        context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    96 96
             context.set_message_handler(message_handler)
    
    97 97
     
    
    98 98
             # Load the project and CAS cache
    
    ... ... @@ -102,7 +102,10 @@ def test_pull(cli, tmpdir, datafiles):
    102 102
     
    
    103 103
             # Assert that the element's artifact is **not** cached
    
    104 104
             element = project.load_elements(['target.bst'])[0]
    
    105
    +        print(element)
    
    105 106
             element_key = cli.get_element_key(project_dir, 'target.bst')
    
    107
    +        print(context.casdir)
    
    108
    +        print(cas.get_artifact_fullname(element, element_key))
    
    106 109
             assert not cas.contains(element, element_key)
    
    107 110
     
    
    108 111
             queue = multiprocessing.Queue()
    
    ... ... @@ -110,7 +113,7 @@ def test_pull(cli, tmpdir, datafiles):
    110 113
             # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
    
    111 114
             process = multiprocessing.Process(target=_queue_wrapper,
    
    112 115
                                               args=(_test_pull, queue, user_config_file, project_dir,
    
    113
    -                                                artifact_dir, 'target.bst', element_key))
    
    116
    +                                                cache_dir, 'target.bst', element_key))
    
    114 117
     
    
    115 118
             try:
    
    116 119
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -127,12 +130,14 @@ def test_pull(cli, tmpdir, datafiles):
    127 130
             assert cas.contains(element, element_key)
    
    128 131
     
    
    129 132
     
    
    130
    -def _test_pull(user_config_file, project_dir, artifact_dir,
    
    133
    +def _test_pull(user_config_file, project_dir, cache_dir,
    
    131 134
                    element_name, element_key, queue):
    
    132 135
         # Fake minimal context
    
    133 136
         context = Context()
    
    134 137
         context.load(config=user_config_file)
    
    135
    -    context.artifactdir = artifact_dir
    
    138
    +    context.rootcachedir = cache_dir
    
    139
    +    context.casdir = os.path.join(cache_dir, 'cas')
    
    140
    +    context.tmpdir = os.path.join(cache_dir, 'tmp')
    
    136 141
         context.set_message_handler(message_handler)
    
    137 142
     
    
    138 143
         # Load the project manually
    
    ... ... @@ -165,7 +170,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
    165 170
         # Set up an artifact cache.
    
    166 171
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    167 172
             # Configure artifact share
    
    168
    -        artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    173
    +        rootcache_dir = os.path.join(str(tmpdir), 'cache')
    
    169 174
             user_config_file = str(tmpdir.join('buildstream.conf'))
    
    170 175
             user_config = {
    
    171 176
                 'scheduler': {
    
    ... ... @@ -174,7 +179,8 @@ def test_pull_tree(cli, tmpdir, datafiles):
    174 179
                 'artifacts': {
    
    175 180
                     'url': share.repo,
    
    176 181
                     'push': True,
    
    177
    -            }
    
    182
    +            },
    
    183
    +            'rootcachedir': rootcache_dir
    
    178 184
             }
    
    179 185
     
    
    180 186
             # Write down the user configuration file
    
    ... ... @@ -195,7 +201,6 @@ def test_pull_tree(cli, tmpdir, datafiles):
    195 201
             # Fake minimal context
    
    196 202
             context = Context()
    
    197 203
             context.load(config=user_config_file)
    
    198
    -        context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    199 204
             context.set_message_handler(message_handler)
    
    200 205
     
    
    201 206
             # Load the project and CAS cache
    
    ... ... @@ -218,7 +223,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
    218 223
             # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
    
    219 224
             process = multiprocessing.Process(target=_queue_wrapper,
    
    220 225
                                               args=(_test_push_tree, queue, user_config_file, project_dir,
    
    221
    -                                                artifact_dir, artifact_digest))
    
    226
    +                                                artifact_digest))
    
    222 227
     
    
    223 228
             try:
    
    224 229
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -246,7 +251,7 @@ def test_pull_tree(cli, tmpdir, datafiles):
    246 251
             # Use subprocess to avoid creation of gRPC threads in main BuildStream process
    
    247 252
             process = multiprocessing.Process(target=_queue_wrapper,
    
    248 253
                                               args=(_test_pull_tree, queue, user_config_file, project_dir,
    
    249
    -                                                artifact_dir, tree_digest))
    
    254
    +                                                tree_digest))
    
    250 255
     
    
    251 256
             try:
    
    252 257
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -268,11 +273,10 @@ def test_pull_tree(cli, tmpdir, datafiles):
    268 273
             assert os.path.exists(cas.objpath(directory_digest))
    
    269 274
     
    
    270 275
     
    
    271
    -def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
    
    276
    +def _test_push_tree(user_config_file, project_dir, artifact_digest, queue):
    
    272 277
         # Fake minimal context
    
    273 278
         context = Context()
    
    274 279
         context.load(config=user_config_file)
    
    275
    -    context.artifactdir = artifact_dir
    
    276 280
         context.set_message_handler(message_handler)
    
    277 281
     
    
    278 282
         # Load the project manually
    
    ... ... @@ -304,11 +308,10 @@ def _test_push_tree(user_config_file, project_dir, artifact_dir, artifact_digest
    304 308
             queue.put("No remote configured")
    
    305 309
     
    
    306 310
     
    
    307
    -def _test_pull_tree(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
    
    311
    +def _test_pull_tree(user_config_file, project_dir, artifact_digest, queue):
    
    308 312
         # Fake minimal context
    
    309 313
         context = Context()
    
    310 314
         context.load(config=user_config_file)
    
    311
    -    context.artifactdir = artifact_dir
    
    312 315
         context.set_message_handler(message_handler)
    
    313 316
     
    
    314 317
         # Load the project manually
    

  • tests/artifactcache/push.py
    ... ... @@ -51,7 +51,7 @@ def test_push(cli, tmpdir, datafiles):
    51 51
         # Set up an artifact cache.
    
    52 52
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    53 53
             # Configure artifact share
    
    54
    -        artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    54
    +        rootcache_dir = os.path.join(str(tmpdir), 'cache')
    
    55 55
             user_config_file = str(tmpdir.join('buildstream.conf'))
    
    56 56
             user_config = {
    
    57 57
                 'scheduler': {
    
    ... ... @@ -60,7 +60,8 @@ def test_push(cli, tmpdir, datafiles):
    60 60
                 'artifacts': {
    
    61 61
                     'url': share.repo,
    
    62 62
                     'push': True,
    
    63
    -            }
    
    63
    +            },
    
    64
    +            'rootcachedir': rootcache_dir
    
    64 65
             }
    
    65 66
     
    
    66 67
             # Write down the user configuration file
    
    ... ... @@ -69,7 +70,6 @@ def test_push(cli, tmpdir, datafiles):
    69 70
             # Fake minimal context
    
    70 71
             context = Context()
    
    71 72
             context.load(config=user_config_file)
    
    72
    -        context.artifactdir = artifact_dir
    
    73 73
             context.set_message_handler(message_handler)
    
    74 74
     
    
    75 75
             # Load the project manually
    
    ... ... @@ -89,7 +89,7 @@ def test_push(cli, tmpdir, datafiles):
    89 89
             # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
    
    90 90
             process = multiprocessing.Process(target=_queue_wrapper,
    
    91 91
                                               args=(_test_push, queue, user_config_file, project_dir,
    
    92
    -                                                artifact_dir, 'target.bst', element_key))
    
    92
    +                                                'target.bst', element_key))
    
    93 93
     
    
    94 94
             try:
    
    95 95
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -106,12 +106,10 @@ def test_push(cli, tmpdir, datafiles):
    106 106
             assert share.has_artifact('test', 'target.bst', element_key)
    
    107 107
     
    
    108 108
     
    
    109
    -def _test_push(user_config_file, project_dir, artifact_dir,
    
    110
    -               element_name, element_key, queue):
    
    109
    +def _test_push(user_config_file, project_dir, element_name, element_key, queue):
    
    111 110
         # Fake minimal context
    
    112 111
         context = Context()
    
    113 112
         context.load(config=user_config_file)
    
    114
    -    context.artifactdir = artifact_dir
    
    115 113
         context.set_message_handler(message_handler)
    
    116 114
     
    
    117 115
         # Load the project manually
    
    ... ... @@ -152,7 +150,7 @@ def test_push_directory(cli, tmpdir, datafiles):
    152 150
         # Set up an artifact cache.
    
    153 151
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    154 152
             # Configure artifact share
    
    155
    -        artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    153
    +        rootcache_dir = os.path.join(str(tmpdir), 'cache')
    
    156 154
             user_config_file = str(tmpdir.join('buildstream.conf'))
    
    157 155
             user_config = {
    
    158 156
                 'scheduler': {
    
    ... ... @@ -161,7 +159,8 @@ def test_push_directory(cli, tmpdir, datafiles):
    161 159
                 'artifacts': {
    
    162 160
                     'url': share.repo,
    
    163 161
                     'push': True,
    
    164
    -            }
    
    162
    +            },
    
    163
    +            'rootcachedir': rootcache_dir
    
    165 164
             }
    
    166 165
     
    
    167 166
             # Write down the user configuration file
    
    ... ... @@ -170,7 +169,6 @@ def test_push_directory(cli, tmpdir, datafiles):
    170 169
             # Fake minimal context
    
    171 170
             context = Context()
    
    172 171
             context.load(config=user_config_file)
    
    173
    -        context.artifactdir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    174 172
             context.set_message_handler(message_handler)
    
    175 173
     
    
    176 174
             # Load the project and CAS cache
    
    ... ... @@ -182,6 +180,7 @@ def test_push_directory(cli, tmpdir, datafiles):
    182 180
             # Assert that the element's artifact is cached
    
    183 181
             element = project.load_elements(['target.bst'])[0]
    
    184 182
             element_key = cli.get_element_key(project_dir, 'target.bst')
    
    183
    +        print(context.casdir)
    
    185 184
             assert artifactcache.contains(element, element_key)
    
    186 185
     
    
    187 186
             # Manually setup the CAS remote
    
    ... ... @@ -198,7 +197,7 @@ def test_push_directory(cli, tmpdir, datafiles):
    198 197
             # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
    
    199 198
             process = multiprocessing.Process(target=_queue_wrapper,
    
    200 199
                                               args=(_test_push_directory, queue, user_config_file,
    
    201
    -                                                project_dir, artifact_dir, artifact_digest))
    
    200
    +                                                project_dir, artifact_digest))
    
    202 201
     
    
    203 202
             try:
    
    204 203
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -216,11 +215,10 @@ def test_push_directory(cli, tmpdir, datafiles):
    216 215
             assert share.has_object(artifact_digest)
    
    217 216
     
    
    218 217
     
    
    219
    -def _test_push_directory(user_config_file, project_dir, artifact_dir, artifact_digest, queue):
    
    218
    +def _test_push_directory(user_config_file, project_dir, artifact_digest, queue):
    
    220 219
         # Fake minimal context
    
    221 220
         context = Context()
    
    222 221
         context.load(config=user_config_file)
    
    223
    -    context.artifactdir = artifact_dir
    
    224 222
         context.set_message_handler(message_handler)
    
    225 223
     
    
    226 224
         # Load the project manually
    
    ... ... @@ -254,6 +252,7 @@ def test_push_message(cli, tmpdir, datafiles):
    254 252
         with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share:
    
    255 253
             # Configure artifact share
    
    256 254
             artifact_dir = os.path.join(str(tmpdir), 'cache', 'artifacts')
    
    255
    +        rootcache_dir = os.path.join(str(tmpdir), 'cache')
    
    257 256
             user_config_file = str(tmpdir.join('buildstream.conf'))
    
    258 257
             user_config = {
    
    259 258
                 'scheduler': {
    
    ... ... @@ -262,7 +261,8 @@ def test_push_message(cli, tmpdir, datafiles):
    262 261
                 'artifacts': {
    
    263 262
                     'url': share.repo,
    
    264 263
                     'push': True,
    
    265
    -            }
    
    264
    +            },
    
    265
    +            'rootcachedir': rootcache_dir
    
    266 266
             }
    
    267 267
     
    
    268 268
             # Write down the user configuration file
    
    ... ... @@ -273,7 +273,7 @@ def test_push_message(cli, tmpdir, datafiles):
    273 273
             # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
    
    274 274
             process = multiprocessing.Process(target=_queue_wrapper,
    
    275 275
                                               args=(_test_push_message, queue, user_config_file,
    
    276
    -                                                project_dir, artifact_dir))
    
    276
    +                                                project_dir))
    
    277 277
     
    
    278 278
             try:
    
    279 279
                 # Keep SIGINT blocked in the child process
    
    ... ... @@ -292,11 +292,10 @@ def test_push_message(cli, tmpdir, datafiles):
    292 292
             assert share.has_object(message_digest)
    
    293 293
     
    
    294 294
     
    
    295
    -def _test_push_message(user_config_file, project_dir, artifact_dir, queue):
    
    295
    +def _test_push_message(user_config_file, project_dir, queue):
    
    296 296
         # Fake minimal context
    
    297 297
         context = Context()
    
    298 298
         context.load(config=user_config_file)
    
    299
    -    context.artifactdir = artifact_dir
    
    300 299
         context.set_message_handler(message_handler)
    
    301 300
     
    
    302 301
         # Load the project manually
    

  • tests/frontend/pull.py
    ... ... @@ -62,8 +62,8 @@ def test_push_pull_all(cli, tmpdir, datafiles):
    62 62
             # Now we've pushed, delete the user's local artifact cache
    
    63 63
             # directory and try to redownload it from the share
    
    64 64
             #
    
    65
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    66
    -        shutil.rmtree(artifacts)
    
    65
    +        cas = os.path.join(cli.directory, 'cas')
    
    66
    +        shutil.rmtree(cas)
    
    67 67
     
    
    68 68
             # Assert that nothing is cached locally anymore
    
    69 69
             for element_name in all_elements:
    
    ... ... @@ -104,8 +104,8 @@ def test_pull_secondary_cache(cli, tmpdir, datafiles):
    104 104
             assert_shared(cli, share2, project, 'target.bst')
    
    105 105
     
    
    106 106
             # Delete the user's local artifact cache.
    
    107
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    108
    -        shutil.rmtree(artifacts)
    
    107
    +        cas = os.path.join(cli.directory, 'cas')
    
    108
    +        shutil.rmtree(cas)
    
    109 109
     
    
    110 110
             # Assert that the element is not cached anymore.
    
    111 111
             assert cli.get_element_state(project, 'target.bst') != 'cached'
    
    ... ... @@ -158,8 +158,8 @@ def test_push_pull_specific_remote(cli, tmpdir, datafiles):
    158 158
             # Now we've pushed, delete the user's local artifact cache
    
    159 159
             # directory and try to redownload it from the good_share.
    
    160 160
             #
    
    161
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    162
    -        shutil.rmtree(artifacts)
    
    161
    +        cas = os.path.join(cli.directory, 'cas')
    
    162
    +        shutil.rmtree(cas)
    
    163 163
     
    
    164 164
             result = cli.run(project=project, args=['artifact', 'pull', 'target.bst', '--remote',
    
    165 165
                                                     good_share.repo])
    
    ... ... @@ -199,8 +199,8 @@ def test_push_pull_non_strict(cli, tmpdir, datafiles):
    199 199
             # Now we've pushed, delete the user's local artifact cache
    
    200 200
             # directory and try to redownload it from the share
    
    201 201
             #
    
    202
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    203
    -        shutil.rmtree(artifacts)
    
    202
    +        cas = os.path.join(cli.directory, 'cas')
    
    203
    +        shutil.rmtree(cas)
    
    204 204
     
    
    205 205
             # Assert that nothing is cached locally anymore
    
    206 206
             for element_name in all_elements:
    
    ... ... @@ -249,8 +249,8 @@ def test_push_pull_track_non_strict(cli, tmpdir, datafiles):
    249 249
             # Now we've pushed, delete the user's local artifact cache
    
    250 250
             # directory and try to redownload it from the share
    
    251 251
             #
    
    252
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    253
    -        shutil.rmtree(artifacts)
    
    252
    +        cas = os.path.join(cli.directory, 'cas')
    
    253
    +        shutil.rmtree(cas)
    
    254 254
     
    
    255 255
             # Assert that nothing is cached locally anymore
    
    256 256
             for element_name in all_elements:
    
    ... ... @@ -285,7 +285,7 @@ def test_push_pull_cross_junction(cli, tmpdir, datafiles):
    285 285
             result.assert_success()
    
    286 286
             assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'cached'
    
    287 287
     
    
    288
    -        cache_dir = os.path.join(project, 'cache', 'artifacts')
    
    288
    +        cache_dir = os.path.join(project, 'cache', 'cas')
    
    289 289
             shutil.rmtree(cache_dir)
    
    290 290
     
    
    291 291
             assert cli.get_element_state(project, 'junction.bst:import-etc.bst') == 'buildable'
    
    ... ... @@ -320,8 +320,8 @@ def test_pull_missing_blob(cli, tmpdir, datafiles):
    320 320
             # Now we've pushed, delete the user's local artifact cache
    
    321 321
             # directory and try to redownload it from the share
    
    322 322
             #
    
    323
    -        artifacts = os.path.join(cli.directory, 'artifacts')
    
    324
    -        shutil.rmtree(artifacts)
    
    323
    +        cas = os.path.join(cli.directory, 'cas')
    
    324
    +        shutil.rmtree(cas)
    
    325 325
     
    
    326 326
             # Assert that nothing is cached locally anymore
    
    327 327
             for element_name in all_elements:
    

  • tests/integration/build-tree.py
    ... ... @@ -158,10 +158,8 @@ def test_buildtree_options(cli, tmpdir, datafiles):
    158 158
             assert cli.get_element_state(project, element_name) == 'cached'
    
    159 159
     
    
    160 160
             # Discard the cache
    
    161
    -        cli.configure({
    
    162
    -            'artifacts': {'url': share.repo, 'push': True},
    
    163
    -            'artifactdir': os.path.join(cli.directory, 'artifacts2')
    
    164
    -        })
    
    161
    +        shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts')))
    
    162
    +        shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas')))
    
    165 163
             assert cli.get_element_state(project, element_name) != 'cached'
    
    166 164
     
    
    167 165
             # Pull from cache, but do not include buildtrees.
    

  • tests/integration/cachedfail.py
    ... ... @@ -160,7 +160,6 @@ def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
    160 160
     
    
    161 161
             # This element should have failed
    
    162 162
             assert cli.get_element_state(project, 'element.bst') == 'failed'
    
    163
    -        # This element should have been pushed to the remote
    
    164 163
             assert share.has_artifact('test', 'element.bst', cli.get_element_key(project, 'element.bst'))
    
    165 164
     
    
    166 165
     
    

  • tests/integration/messages.py
    ... ... @@ -40,7 +40,7 @@ DATA_DIR = os.path.join(
    40 40
     @pytest.mark.integration
    
    41 41
     @pytest.mark.datafiles(DATA_DIR)
    
    42 42
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    43
    -def test_disable_message_lines(cli, tmpdir, datafiles):
    
    43
    +def test_disable_message_lines(cli, tmpdir, datafiles, integration_cache):
    
    44 44
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    45 45
         element_path = os.path.join(project, 'elements')
    
    46 46
         element_name = 'message.bst'
    
    ... ... @@ -66,7 +66,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
    66 66
         assert 'echo "Silly message"' in result.stderr
    
    67 67
     
    
    68 68
         # Let's now build it again, but with --message-lines 0
    
    69
    -    cli.remove_artifact_from_cache(project, element_name)
    
    69
    +    cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
    
    70 70
         result = cli.run(project=project, args=["--message-lines", "0",
    
    71 71
                                                 "build", element_name])
    
    72 72
         result.assert_success()
    
    ... ... @@ -76,7 +76,7 @@ def test_disable_message_lines(cli, tmpdir, datafiles):
    76 76
     @pytest.mark.integration
    
    77 77
     @pytest.mark.datafiles(DATA_DIR)
    
    78 78
     @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
    
    79
    -def test_disable_error_lines(cli, tmpdir, datafiles):
    
    79
    +def test_disable_error_lines(cli, tmpdir, datafiles, integration_cache):
    
    80 80
         project = os.path.join(datafiles.dirname, datafiles.basename)
    
    81 81
         element_path = os.path.join(project, 'elements')
    
    82 82
         element_name = 'message.bst'
    
    ... ... @@ -103,7 +103,7 @@ def test_disable_error_lines(cli, tmpdir, datafiles):
    103 103
         assert "This is a syntax error" in result.stderr
    
    104 104
     
    
    105 105
         # Let's now build it again, but with --error-lines 0
    
    106
    -    cli.remove_artifact_from_cache(project, element_name)
    
    106
    +    cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
    
    107 107
         result = cli.run(project=project, args=["--error-lines", "0",
    
    108 108
                                                 "build", element_name])
    
    109 109
         result.assert_main_error(ErrorDomain.STREAM, None)
    

  • tests/integration/pullbuildtrees.py
    ... ... @@ -19,9 +19,10 @@ DATA_DIR = os.path.join(
    19 19
     # cleared as just forcefully removing the refpath leaves dangling objects.
    
    20 20
     def default_state(cli, tmpdir, share):
    
    21 21
         shutil.rmtree(os.path.join(str(tmpdir), 'artifacts'))
    
    22
    +    shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
    
    22 23
         cli.configure({
    
    23 24
             'artifacts': {'url': share.repo, 'push': False},
    
    24
    -        'artifactdir': os.path.join(str(tmpdir), 'artifacts'),
    
    25
    +        'rootcachedir': str(tmpdir),
    
    25 26
             'cache': {'pull-buildtrees': False},
    
    26 27
         })
    
    27 28
     
    
    ... ... @@ -42,7 +43,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
    42 43
             create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
    
    43 44
             cli2.configure({
    
    44 45
                 'artifacts': {'url': share1.repo, 'push': True},
    
    45
    -            'artifactdir': os.path.join(str(tmpdir), 'artifacts')
    
    46
    +            'rootcachedir': str(tmpdir),
    
    46 47
             })
    
    47 48
     
    
    48 49
             # Build autotools element, checked pushed, delete local
    
    ... ... @@ -79,7 +80,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
    79 80
             assert os.path.isdir(buildtreedir)
    
    80 81
             default_state(cli2, tmpdir, share1)
    
    81 82
     
    
    82
    -        # Pull artifact with pullbuildtrees set in user config, then assert
    
    83
    +        # Pull artifact with pullbuildtrees set in user onfig, then assert
    
    83 84
             # that pulling with the same user config doesn't creates a pull job,
    
    84 85
             # or when buildtrees cli flag is set.
    
    85 86
             cli2.configure({'cache': {'pull-buildtrees': True}})
    

  • tests/integration/source-determinism.py
    ... ... @@ -94,9 +94,7 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_ca
    94 94
                     return f.read()
    
    95 95
             finally:
    
    96 96
                 os.umask(old_umask)
    
    97
    -            cache_dir = integration_cache.artifacts
    
    98
    -            cli.remove_artifact_from_cache(project, element_name,
    
    99
    -                                           cache_dir=cache_dir)
    
    97
    +            cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
    
    100 98
     
    
    101 99
         assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    
    102 100
     
    
    ... ... @@ -156,8 +154,6 @@ def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
    156 154
                 with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    157 155
                     return f.read()
    
    158 156
             finally:
    
    159
    -            cache_dir = integration_cache.artifacts
    
    160
    -            cli.remove_artifact_from_cache(project, element_name,
    
    161
    -                                           cache_dir=cache_dir)
    
    157
    +            cli.remove_artifact_from_cache(project, element_name, cache_dir=integration_cache.root)
    
    162 158
     
    
    163 159
         assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

  • tests/testutils/runcli.py
    ... ... @@ -247,15 +247,13 @@ class Cli():
    247 247
                                        *, cache_dir=None):
    
    248 248
             # Read configuration to figure out where artifacts are stored
    
    249 249
             if not cache_dir:
    
    250
    -            default = os.path.join(project, 'cache', 'artifacts')
    
    251
    -
    
    252
    -            if self.config is not None:
    
    253
    -                cache_dir = self.config.get('artifactdir', default)
    
    254
    -            else:
    
    255
    -                cache_dir = default
    
    250
    +            cache_dir = os.path.join(project, 'cache')
    
    256 251
     
    
    257 252
             cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
    
    258 253
     
    
    254
    +        # replace forward slashes
    
    255
    +        element_name = element_name.replace('/', '-')
    
    256
    +
    
    259 257
             cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
    
    260 258
             shutil.rmtree(cache_dir)
    
    261 259
     
    
    ... ... @@ -553,11 +551,21 @@ def cli_integration(tmpdir, integration_cache):
    553 551
         # We want to cache sources for integration tests more permanently,
    
    554 552
         # to avoid downloading the huge base-sdk repeatedly
    
    555 553
         fixture.configure({
    
    554
    +        'rootcachedir': integration_cache.root,
    
    556 555
             'sourcedir': integration_cache.sources,
    
    557
    -        'artifactdir': integration_cache.artifacts
    
    558 556
         })
    
    559 557
     
    
    560
    -    return fixture
    
    558
    +    yield fixture
    
    559
    +
    
    560
    +    # remove following folders if necessary
    
    561
    +    try:
    
    562
    +        shutil.rmtree(os.path.join(integration_cache.root, 'build'))
    
    563
    +    except FileNotFoundError:
    
    564
    +        pass
    
    565
    +    try:
    
    566
    +        shutil.rmtree(os.path.join(integration_cache.root, 'tmp'))
    
    567
    +    except FileNotFoundError:
    
    568
    +        pass
    
    561 569
     
    
    562 570
     
    
    563 571
     @contextmanager
    
    ... ... @@ -597,10 +605,8 @@ def configured(directory, config=None):
    597 605
     
    
    598 606
         if not config.get('sourcedir', False):
    
    599 607
             config['sourcedir'] = os.path.join(directory, 'sources')
    
    600
    -    if not config.get('builddir', False):
    
    601
    -        config['builddir'] = os.path.join(directory, 'build')
    
    602
    -    if not config.get('artifactdir', False):
    
    603
    -        config['artifactdir'] = os.path.join(directory, 'artifacts')
    
    608
    +    if not config.get('rootcachedir', False):
    
    609
    +        config['rootcachedir'] = directory
    
    604 610
         if not config.get('logdir', False):
    
    605 611
             config['logdir'] = os.path.join(directory, 'logs')
    
    606 612
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]