[Notes] [Git][BuildStream/buildstream][chandan/setup.py-metadata] 13 commits: .gitlab-ci.yml: Verify that source tarball installs correctly



Title: GitLab

Chandan Singh pushed to branch chandan/setup.py-metadata at BuildStream / buildstream

Commits:

11 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -36,6 +36,11 @@ source_dist:
    36 36
       - tar -ztf dist/*
    
    37 37
       - tarball=$(cd dist && echo $(ls *))
    
    38 38
     
    
    39
    +  # Verify that the source distribution tarball can be installed correctly
    
    40
    +  #
    
    41
    +  - pip3 install dist/*.tar.gz
    
    42
    +  - bst --version
    
    43
    +
    
    39 44
       # unpack tarball as `dist/buildstream` directory
    
    40 45
       - |
    
    41 46
         cat > dist/unpack.sh << EOF
    

  • HACKING.rst
    ... ... @@ -465,12 +465,12 @@ If you want to run a specific test or a group of tests, you
    465 465
     can specify a prefix to match. E.g. if you want to run all of
    
    466 466
     the frontend tests you can do::
    
    467 467
     
    
    468
    -  ./setup.py test --addopts '-k tests/frontend/'
    
    468
    +  ./setup.py test --addopts 'tests/frontend/'
    
    469 469
     
    
    470 470
     Specific tests can be chosen by using the :: delimeter after the test module.
    
    471 471
     If you wanted to run the test_build_track test within frontend/buildtrack.py you could do::
    
    472 472
     
    
    473
    -  ./setup.py test --adopts '-k tests/frontend/buildtrack.py::test_build_track'
    
    473
    +  ./setup.py test --addopts 'tests/frontend/buildtrack.py::test_build_track'
    
    474 474
     
    
    475 475
     We also have a set of slow integration tests that are disabled by
    
    476 476
     default - you will notice most of them marked with SKIP in the pytest
    

  • buildstream/element.py
    ... ... @@ -1361,8 +1361,12 @@ class Element(Plugin):
    1361 1361
                 if not vdirectory.is_empty():
    
    1362 1362
                     raise ElementError("Staging directory '{}' is not empty".format(vdirectory))
    
    1363 1363
     
    
    1364
    -            with tempfile.TemporaryDirectory() as temp_staging_directory:
    
    1364
    +            # While mkdtemp is advertised as using the TMP environment variable, it
    
    1365
    +            # doesn't, so this explicit extraction is necesasry.
    
    1366
    +            tmp_prefix = os.environ.get("TMP", None)
    
    1367
    +            temp_staging_directory = tempfile.mkdtemp(prefix=tmp_prefix)
    
    1365 1368
     
    
    1369
    +            try:
    
    1366 1370
                     workspace = self._get_workspace()
    
    1367 1371
                     if workspace:
    
    1368 1372
                         # If mount_workspaces is set and we're doing incremental builds,
    
    ... ... @@ -1377,6 +1381,19 @@ class Element(Plugin):
    1377 1381
                             source._stage(temp_staging_directory)
    
    1378 1382
     
    
    1379 1383
                     vdirectory.import_files(temp_staging_directory)
    
    1384
    +
    
    1385
    +            finally:
    
    1386
    +                # Staging may produce directories with less than 'rwx' permissions
    
    1387
    +                # for the owner, which will break tempfile, so we need to use chmod
    
    1388
    +                # occasionally.
    
    1389
    +                def make_dir_writable(fn, path, excinfo):
    
    1390
    +                    os.chmod(os.path.dirname(path), 0o777)
    
    1391
    +                    if os.path.isdir(path):
    
    1392
    +                        os.rmdir(path)
    
    1393
    +                    else:
    
    1394
    +                        os.remove(path)
    
    1395
    +                shutil.rmtree(temp_staging_directory, onerror=make_dir_writable)
    
    1396
    +
    
    1380 1397
             # Ensure deterministic mtime of sources at build time
    
    1381 1398
             vdirectory.set_deterministic_mtime()
    
    1382 1399
             # Ensure deterministic owners of sources at build time
    

  • setup.py
    ... ... @@ -242,11 +242,28 @@ setup(name='BuildStream',
    242 242
     
    
    243 243
           author='BuildStream Developers',
    
    244 244
           author_email='buildstream-list gnome org',
    
    245
    +      classifiers=[
    
    246
    +          'Environment :: Console',
    
    247
    +          'Intended Audience :: Developers',
    
    248
    +          'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
    
    249
    +          'Operating System :: POSIX',
    
    250
    +          'Programming Language :: Python :: 3',
    
    251
    +          'Programming Language :: Python :: 3.5',
    
    252
    +          'Programming Language :: Python :: 3.6',
    
    253
    +          'Programming Language :: Python :: 3.7',
    
    254
    +          'Topic :: Software Development :: Build Tools'
    
    255
    +      ],
    
    245 256
           description='A framework for modelling build pipelines in YAML',
    
    246 257
           license='LGPL',
    
    247 258
           long_description=long_description,
    
    248 259
           long_description_content_type='text/x-rst; charset=UTF-8',
    
    249 260
           url='https://gitlab.com/BuildStream/buildstream',
    
    261
    +      project_urls={
    
    262
    +          'Documentation': 'https://buildstream.gitlab.io/buildstream/',
    
    263
    +          'Tracker': 'https://gitlab.com/BuildStream/buildstream/issues',
    
    264
    +          'Mailing List': 'https://mail.gnome.org/mailman/listinfo/buildstream-list'
    
    265
    +      },
    
    266
    +      python_requires='>=3.5',
    
    250 267
           packages=find_packages(exclude=('tests', 'tests.*')),
    
    251 268
           package_data={'buildstream': ['plugins/*/*.py', 'plugins/*/*.yaml',
    
    252 269
                                         'data/*.yaml', 'data/*.sh.in']},
    

  • tests/integration/project/elements/base/base-alpine.bst
    ... ... @@ -7,6 +7,6 @@ description: |
    7 7
     
    
    8 8
     sources:
    
    9 9
       - kind: tar
    
    10
    -    url: sysroot:tarballs/integration-tests-base.v1.x86_64.tar.xz
    
    10
    +    url: alpine:integration-tests-base.v1.x86_64.tar.xz
    
    11 11
         base-dir: ''
    
    12 12
         ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

  • tests/integration/project/project.conf
    ... ... @@ -2,7 +2,7 @@
    2 2
     name: test
    
    3 3
     element-path: elements
    
    4 4
     aliases:
    
    5
    -  sysroot: https://gnome7.codethink.co.uk/
    
    5
    +  alpine: https://gnome7.codethink.co.uk/tarballs/
    
    6 6
       project_dir: file://{project_dir}
    
    7 7
     options:
    
    8 8
       linux:
    

  • tests/integration/source-determinism.py
    ... ... @@ -2,7 +2,8 @@ import os
    2 2
     import pytest
    
    3 3
     
    
    4 4
     from buildstream import _yaml, utils
    
    5
    -from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    
    5
    +from tests.testutils import create_repo, ALL_REPO_KINDS
    
    6
    +from tests.testutils import cli_integration as cli
    
    6 7
     
    
    7 8
     
    
    8 9
     DATA_DIR = os.path.join(
    
    ... ... @@ -28,7 +29,7 @@ def create_test_directory(*path, mode=0o644):
    28 29
     @pytest.mark.integration
    
    29 30
     @pytest.mark.datafiles(DATA_DIR)
    
    30 31
     @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    
    31
    -def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    
    32
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind, integration_cache):
    
    32 33
         project = str(datafiles)
    
    33 34
         element_name = 'list'
    
    34 35
         element_path = os.path.join(project, 'elements', element_name)
    
    ... ... @@ -91,14 +92,16 @@ def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    91 92
                     return f.read()
    
    92 93
             finally:
    
    93 94
                 os.umask(old_umask)
    
    94
    -            cli.remove_artifact_from_cache(project, element_name)
    
    95
    +            cache_dir = os.path.join(integration_cache, 'artifacts')
    
    96
    +            cli.remove_artifact_from_cache(project, element_name,
    
    97
    +                                           cache_dir=cache_dir)
    
    95 98
     
    
    96 99
         assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    
    97 100
     
    
    98 101
     
    
    99 102
     @pytest.mark.integration
    
    100 103
     @pytest.mark.datafiles(DATA_DIR)
    
    101
    -def test_deterministic_source_local(cli, tmpdir, datafiles):
    
    104
    +def test_deterministic_source_local(cli, tmpdir, datafiles, integration_cache):
    
    102 105
         """Only user rights should be considered for local source.
    
    103 106
         """
    
    104 107
         project = str(datafiles)
    
    ... ... @@ -150,6 +153,8 @@ def test_deterministic_source_local(cli, tmpdir, datafiles):
    150 153
                 with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    151 154
                     return f.read()
    
    152 155
             finally:
    
    153
    -            cli.remove_artifact_from_cache(project, element_name)
    
    156
    +            cache_dir = os.path.join(integration_cache, 'artifacts')
    
    157
    +            cli.remove_artifact_from_cache(project, element_name,
    
    158
    +                                           cache_dir=cache_dir)
    
    154 159
     
    
    155 160
         assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

  • tests/sources/tar.py
    ... ... @@ -3,6 +3,7 @@ import pytest
    3 3
     import tarfile
    
    4 4
     import tempfile
    
    5 5
     import subprocess
    
    6
    +from shutil import copyfile, rmtree
    
    6 7
     
    
    7 8
     from buildstream._exceptions import ErrorDomain
    
    8 9
     from buildstream import _yaml
    
    ... ... @@ -257,3 +258,47 @@ def test_stage_default_basedir_lzip(cli, tmpdir, datafiles, srcdir):
    257 258
         original_contents = list_dir_contents(original_dir)
    
    258 259
         checkout_contents = list_dir_contents(checkoutdir)
    
    259 260
         assert(checkout_contents == original_contents)
    
    261
    +
    
    262
    +
    
    263
    +# Test that a tarball that contains a read only dir works
    
    264
    +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'read-only'))
    
    265
    +def test_read_only_dir(cli, tmpdir, datafiles):
    
    266
    +    try:
    
    267
    +        project = os.path.join(datafiles.dirname, datafiles.basename)
    
    268
    +        generate_project(project, tmpdir)
    
    269
    +
    
    270
    +        # Get the tarball in tests/sources/tar/read-only/content
    
    271
    +        #
    
    272
    +        # NOTE that we need to do this because tarfile.open and tar.add()
    
    273
    +        # are packing the tar up with writeable files and dirs
    
    274
    +        tarball = os.path.join(str(datafiles), 'content', 'a.tar.gz')
    
    275
    +        if not os.path.exists(tarball):
    
    276
    +            raise FileNotFoundError('{} does not exist'.format(tarball))
    
    277
    +        copyfile(tarball, os.path.join(str(tmpdir), 'a.tar.gz'))
    
    278
    +
    
    279
    +        # Because this test can potentially leave directories behind
    
    280
    +        # which are difficult to remove, ask buildstream to use
    
    281
    +        # our temp directory, so we can clean up.
    
    282
    +        tmpdir_str = str(tmpdir)
    
    283
    +        if not tmpdir_str.endswith(os.path.sep):
    
    284
    +            tmpdir_str += os.path.sep
    
    285
    +        env = {"TMP": tmpdir_str}
    
    286
    +
    
    287
    +        # Track, fetch, build, checkout
    
    288
    +        result = cli.run(project=project, args=['track', 'target.bst'], env=env)
    
    289
    +        result.assert_success()
    
    290
    +        result = cli.run(project=project, args=['fetch', 'target.bst'], env=env)
    
    291
    +        result.assert_success()
    
    292
    +        result = cli.run(project=project, args=['build', 'target.bst'], env=env)
    
    293
    +        result.assert_success()
    
    294
    +
    
    295
    +    finally:
    
    296
    +
    
    297
    +        # Make tmpdir deletable no matter what happens
    
    298
    +        def make_dir_writable(fn, path, excinfo):
    
    299
    +            os.chmod(os.path.dirname(path), 0o777)
    
    300
    +            if os.path.isdir(path):
    
    301
    +                os.rmdir(path)
    
    302
    +            else:
    
    303
    +                os.remove(path)
    
    304
    +        rmtree(str(tmpdir), onerror=make_dir_writable)

  • tests/sources/tar/read-only/content/a.tar.gz
    No preview for this file type
  • tests/sources/tar/read-only/target.bst
    1
    +kind: import
    
    2
    +description: The kind of this element is irrelevant.
    
    3
    +sources:
    
    4
    +- kind: tar
    
    5
    +  url: tmpdir:/a.tar.gz
    
    6
    +  ref: foo

  • tests/testutils/runcli.py
    ... ... @@ -198,8 +198,10 @@ class Cli():
    198 198
             for key, val in config.items():
    
    199 199
                 self.config[key] = val
    
    200 200
     
    
    201
    -    def remove_artifact_from_cache(self, project, element_name):
    
    202
    -        cache_dir = os.path.join(project, 'cache', 'artifacts')
    
    201
    +    def remove_artifact_from_cache(self, project, element_name,
    
    202
    +                                   *, cache_dir=None):
    
    203
    +        if not cache_dir:
    
    204
    +            cache_dir = os.path.join(project, 'cache', 'artifacts')
    
    203 205
     
    
    204 206
             cache_dir = os.path.join(cache_dir, 'cas', 'refs', 'heads')
    
    205 207
     
    



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]