[Notes] [Git][BuildStream/buildstream][master] 6 commits: buildstream/plugins/sources/local.py: Make staging deterministic.



Title: GitLab

Tristan Van Berkom pushed to branch master at BuildStream / buildstream

Commits:

30 changed files:

Changes:

  • buildstream/_versions.py
    ... ... @@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 13
    33 33
     # or if buildstream was changed in a way which can cause
    
    34 34
     # the same cache key to produce something that is no longer
    
    35 35
     # the same.
    
    36
    -BST_CORE_ARTIFACT_VERSION = 3
    36
    +BST_CORE_ARTIFACT_VERSION = 4

  • buildstream/plugins/sources/local.py
    ... ... @@ -37,6 +37,7 @@ local - stage local files and directories
    37 37
     """
    
    38 38
     
    
    39 39
     import os
    
    40
    +import stat
    
    40 41
     from buildstream import Source, Consistency
    
    41 42
     from buildstream import utils
    
    42 43
     
    
    ... ... @@ -94,12 +95,35 @@ class LocalSource(Source):
    94 95
             # Dont use hardlinks to stage sources, they are not write protected
    
    95 96
             # in the sandbox.
    
    96 97
             with self.timed_activity("Staging local files at {}".format(self.path)):
    
    98
    +
    
    97 99
                 if os.path.isdir(self.fullpath):
    
    98
    -                utils.copy_files(self.fullpath, directory)
    
    100
    +                files = list(utils.list_relative_paths(self.fullpath, list_dirs=True))
    
    101
    +                utils.copy_files(self.fullpath, directory, files=files)
    
    99 102
                 else:
    
    100 103
                     destfile = os.path.join(directory, os.path.basename(self.path))
    
    104
    +                files = [os.path.basename(self.path)]
    
    101 105
                     utils.safe_copy(self.fullpath, destfile)
    
    102 106
     
    
    107
    +            for f in files:
    
    108
    +                # Non empty directories are not listed by list_relative_paths
    
    109
    +                dirs = f.split(os.sep)
    
    110
    +                for i in range(1, len(dirs)):
    
    111
    +                    d = os.path.join(directory, *(dirs[:i]))
    
    112
    +                    assert os.path.isdir(d) and not os.path.islink(d)
    
    113
    +                    os.chmod(d, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    
    114
    +
    
    115
    +                path = os.path.join(directory, f)
    
    116
    +                if os.path.islink(path):
    
    117
    +                    pass
    
    118
    +                elif os.path.isdir(path):
    
    119
    +                    os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    
    120
    +                else:
    
    121
    +                    st = os.stat(path)
    
    122
    +                    if st.st_mode & stat.S_IXUSR:
    
    123
    +                        os.chmod(path, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    
    124
    +                    else:
    
    125
    +                        os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    
    126
    +
    
    103 127
     
    
    104 128
     # Create a unique key for a file
    
    105 129
     def unique_key(filename):
    

  • buildstream/plugins/sources/zip.py
    ... ... @@ -49,10 +49,17 @@ zip - stage files from zip archives
    49 49
        # To extract the root of the archive directly, this can be set
    
    50 50
        # to an empty string.
    
    51 51
        base-dir: '*'
    
    52
    +
    
    53
    +.. attention::
    
    54
    +
    
    55
    +   File permissions are not preserved. All extracted directories have
    
    56
    +   permissions 0755 and all extracted files have permissions 0644.
    
    57
    +
    
    52 58
     """
    
    53 59
     
    
    54 60
     import os
    
    55 61
     import zipfile
    
    62
    +import stat
    
    56 63
     
    
    57 64
     from buildstream import SourceError
    
    58 65
     from buildstream import utils
    
    ... ... @@ -74,6 +81,9 @@ class ZipSource(DownloadableFileSource):
    74 81
             return super().get_unique_key() + [self.base_dir]
    
    75 82
     
    
    76 83
         def stage(self, directory):
    
    84
    +        exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
    
    85
    +        noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    
    86
    +
    
    77 87
             try:
    
    78 88
                 with zipfile.ZipFile(self._get_mirror_file()) as archive:
    
    79 89
                     base_dir = None
    
    ... ... @@ -81,9 +91,27 @@ class ZipSource(DownloadableFileSource):
    81 91
                         base_dir = self._find_base_dir(archive, self.base_dir)
    
    82 92
     
    
    83 93
                     if base_dir:
    
    84
    -                    archive.extractall(path=directory, members=self._extract_members(archive, base_dir))
    
    94
    +                    members = self._extract_members(archive, base_dir)
    
    85 95
                     else:
    
    86
    -                    archive.extractall(path=directory)
    
    96
    +                    members = archive.namelist()
    
    97
    +
    
    98
    +                for member in members:
    
    99
    +                    written = archive.extract(member, path=directory)
    
    100
    +
    
    101
    +                    # zipfile.extract might create missing directories
    
    102
    +                    rel = os.path.relpath(written, start=directory)
    
    103
    +                    assert not os.path.isabs(rel)
    
    104
    +                    rel = os.path.dirname(rel)
    
    105
    +                    while rel:
    
    106
    +                        os.chmod(os.path.join(directory, rel), exec_rights)
    
    107
    +                        rel = os.path.dirname(rel)
    
    108
    +
    
    109
    +                    if os.path.islink(written):
    
    110
    +                        pass
    
    111
    +                    elif os.path.isdir(written):
    
    112
    +                        os.chmod(written, exec_rights)
    
    113
    +                    else:
    
    114
    +                        os.chmod(written, noexec_rights)
    
    87 115
     
    
    88 116
             except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
    
    89 117
                 raise SourceError("{}: Error staging source: {}".format(self, e)) from e
    

  • buildstream/utils.py
    ... ... @@ -1010,6 +1010,15 @@ def _call(*popenargs, terminate=False, **kwargs):
    1010 1010
     
    
    1011 1011
         process = None
    
    1012 1012
     
    
    1013
    +    old_preexec_fn = kwargs.get('preexec_fn')
    
    1014
    +    if 'preexec_fn' in kwargs:
    
    1015
    +        del kwargs['preexec_fn']
    
    1016
    +
    
    1017
    +    def preexec_fn():
    
    1018
    +        os.umask(stat.S_IWGRP | stat.S_IWOTH)
    
    1019
    +        if old_preexec_fn is not None:
    
    1020
    +            old_preexec_fn()
    
    1021
    +
    
    1013 1022
         # Handle termination, suspend and resume
    
    1014 1023
         def kill_proc():
    
    1015 1024
             if process:
    
    ... ... @@ -1054,7 +1063,7 @@ def _call(*popenargs, terminate=False, **kwargs):
    1054 1063
                 os.killpg(group_id, signal.SIGCONT)
    
    1055 1064
     
    
    1056 1065
         with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
    
    1057
    -        process = subprocess.Popen(*popenargs, **kwargs)
    
    1066
    +        process = subprocess.Popen(*popenargs, preexec_fn=preexec_fn, **kwargs)
    
    1058 1067
             output, _ = process.communicate()
    
    1059 1068
             exit_code = process.poll()
    
    1060 1069
     
    

  • tests/cachekey/project/elements/build1.expected
    1
    -ba707203aa7dfcaea014e65482058a538910da9934c9f91e3ff0d807613c3586
    \ No newline at end of file
    1
    +594be3eb2a211f706557b156ec4b0ffb3ca256af35bcd0116b97fdb8c942d1c5
    \ No newline at end of file

  • tests/cachekey/project/elements/build2.expected
    1
    -7df53245c07746b2d63be01e631a7fcd5befed165fd50175c40a169f07cc35d4
    \ No newline at end of file
    1
    +5038d37bf1714180d160271e688ec1715d69666ca266ed5b1d880abdee36b310
    \ No newline at end of file

  • tests/cachekey/project/elements/compose1.expected
    1
    -ecc49680860cba13bacde567b7cad6c77887518adb7b4cfcce8652baec9a23c9
    \ No newline at end of file
    1
    +e050172c2d445849a2716720814501f0e294b81d28be8cd911ee294291ec60d0
    \ No newline at end of file

  • tests/cachekey/project/elements/compose2.expected
    1
    -8a845f648d40c69c50528960bcda1ef03477e80f7a5f8cc36853801651de4b27
    \ No newline at end of file
    1
    +ad985f243163aab22576fea63a0b89f7560e361326cb041872c568c5feaabf5c
    \ No newline at end of file

  • tests/cachekey/project/elements/compose3.expected
    1
    -638f0b81a2062a0e9bd4f4b14fdb20fe62891826debefa80a22866d3dcb92862
    \ No newline at end of file
    1
    +ec0fec2c821eb34286e9799a6e8d1f5587f161d1d653bd1dbe385602340d86ae
    \ No newline at end of file

  • tests/cachekey/project/elements/compose4.expected
    1
    -ce19faef9a28bd14739876785c125ed288d74b01e3861ddbea1c7f7a5fb428b2
    \ No newline at end of file
    1
    +7cfe8f6161c00d8cf10114e7458f0b97eb003a41504ae301b24f45d48d42155b
    \ No newline at end of file

  • tests/cachekey/project/elements/compose5.expected
    1
    -c29d00bd91a9f1ef61c4b3279759371c9097e1ebfcd9ddf035c8e4997291e35e
    \ No newline at end of file
    1
    +7f016c3165f2de9161312b74f29513dff7dfdcba5ff8c6897beb5b123eaafd3d
    \ No newline at end of file

  • tests/cachekey/project/elements/import1.expected
    1
    -e12330e4a0bd5456f460f47d05d52f434634a14549f01786b9b975ec6bf622cc
    \ No newline at end of file
    1
    +41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
    \ No newline at end of file

  • tests/cachekey/project/elements/import2.expected
    1
    -55625e5ee18294703fa7005e7688dac4e28cc485b306d534a92c0cf77e434b12
    \ No newline at end of file
    1
    +d657db503460486a6de80d87f15c6b1fa84b0c4dacabed374acdc70172d4761d
    \ No newline at end of file

  • tests/cachekey/project/elements/import3.expected
    1
    -714e19b870e61af8f4cba9c5a948f2d0d16d63d796d568b9ed4d6329546cce53
    \ No newline at end of file
    1
    +8de0293a6231dc661cf7229aa5a2f25abdf9a6d38ff70bd6f2562dae51ff05d3
    \ No newline at end of file

  • tests/cachekey/project/elements/script1.expected
    1
    -958b4d93aed53a150c6e15246596e122140a64bd338232619abf4a8cb1b8a3ba
    \ No newline at end of file
    1
    +d0a6b7d29226b083c404d76d9551a0eee98753058580cd62901f8dfac06ca08d
    \ No newline at end of file

  • tests/cachekey/project/sources/bzr1.expected
    1
    -e688b31f8b79f30d11e5fc1121798776d3f9ee68d2d97bb3029bf809fb95892b
    \ No newline at end of file
    1
    +51415ebc7d72315c5c7704759025d6a9237e786bfe9c2bda8c51e15840c3470a
    \ No newline at end of file

  • tests/cachekey/project/sources/git1.expected
    1
    -40eb6b2bd3783189b72ac8465fcae7c7424f5804fbf229cc8984c86b9c07fd56
    \ No newline at end of file
    1
    +ef9bd728a328301e0b819be7109761aacfb4c87092904306d4117c86f30478a4
    \ No newline at end of file

  • tests/cachekey/project/sources/git2.expected
    1
    -3a85acc6dee15a828bfabc4daa9bc1ea72bafa2293a1d2479a99938afa8ee1ff
    \ No newline at end of file
    1
    +a818930895e164bd342ab786061f4d521b27a4470791f55cc28732fdf92794de
    \ No newline at end of file

  • tests/cachekey/project/sources/local1.expected
    1
    -e12330e4a0bd5456f460f47d05d52f434634a14549f01786b9b975ec6bf622cc
    \ No newline at end of file
    1
    +41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
    \ No newline at end of file

  • tests/cachekey/project/sources/local2.expected
    1
    -e080c2b9f00416050cb5bfc64b7f8d79d8ee5b6b4e34e9375dfe75c4207593e6
    \ No newline at end of file
    1
    +bf3ceaa62d472c10ce1e991e5b07a81ddb5206e043d39e60167292910e6bd31e
    \ No newline at end of file

  • tests/cachekey/project/sources/ostree1.expected
    1
    -4782d65dc4492ddec81163478d10973afff6b69e65e4a74fed43afff06cd854b
    \ No newline at end of file
    1
    +c9dcee5ad0822df19984ba68e2a2068266427ee583e3dd3265f85f0515cf7510
    \ No newline at end of file

  • tests/cachekey/project/sources/patch1.expected
    1
    -7bf30dfdcb7bc546c9065c992f75122ab8033413eb52777a69a32be61eade124
    \ No newline at end of file
    1
    +96f2cf27bb2145290fc85b8cbeb7193738a6d22e1328c384bca7ba1e8754d0fd
    \ No newline at end of file

  • tests/cachekey/project/sources/patch2.expected
    1
    -d7459c7255caca1915aecc9cbd9ee7a3360c7e88d2ed76e050ab7f90653ecfea
    \ No newline at end of file
    1
    +bc7cca687951fbaecb365f39fc9d9d7b0fd509b861c77de71f3639e6f49e25fd
    \ No newline at end of file

  • tests/cachekey/project/sources/patch3.expected
    1
    -7d968e2f0675fd998ca880edadd51074d27390ce81f86df3d1435f333bf5712e
    \ No newline at end of file
    1
    +411a041ef82eb9021b308dbc873328e4cc8774c9b8aa0901ff865764bdf82c51
    \ No newline at end of file

  • tests/cachekey/project/sources/tar1.expected
    1
    -9866109275d09a1c7c2ca882c6a3ed5c93345481f2295fd0afc5cf893f7cb432
    \ No newline at end of file
    1
    +a907dfad1c12f6303e7ed20896e49ba3fd5ef566777e7b47098116ec03e0e1f4
    \ No newline at end of file

  • tests/cachekey/project/sources/tar2.expected
    1
    -ac35ce227347a0f0d17a4ecacf959f24290c3fd2488284bbe2c879ed9bb3eaea
    \ No newline at end of file
    1
    +99865afccb0926ba5bbaa24e0ded7d8353b56fe499511ad6a809580d17abd80e
    \ No newline at end of file

  • tests/cachekey/project/sources/zip1.expected
    1
    -38a1d0a72ca251d445d655eb8b5536b10f29ee33924591577ac7e5f9262cc51f
    \ No newline at end of file
    1
    +23ac1cc41c6a72214b3e59664fe0ef85d909013befd8afde13cf8877510579e4
    \ No newline at end of file

  • tests/cachekey/project/sources/zip2.expected
    1
    -98bbec0b0cc9229f5f11f0a829eb192f041b6ac6e5bdb5fc91c849ce91f1e5cf
    \ No newline at end of file
    1
    +be26d9222bf53589686861ce21391548dd5d6284fdd003ff8a7e39601d6e8bef
    \ No newline at end of file

  • tests/cachekey/project/target.expected
    1
    -29c25f47cf186515a7adbec8a613a8ada9fc125b044299cddf1a372b8b4971b3
    \ No newline at end of file
    1
    +0f64d5abf95ea4d5c8e13978e4d8e52fa707a02c9554247ca70a21d7933c4ede
    \ No newline at end of file

  • tests/integration/source-determinism.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from buildstream import _yaml, utils
    
    5
    +from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    
    6
    +
    
    7
    +
    
    8
    +DATA_DIR = os.path.join(
    
    9
    +    os.path.dirname(os.path.realpath(__file__)),
    
    10
    +    "project"
    
    11
    +)
    
    12
    +
    
    13
    +
    
    14
    +def create_test_file(*path, mode=0o644, content='content\n'):
    
    15
    +    path = os.path.join(*path)
    
    16
    +    os.makedirs(os.path.dirname(path), exist_ok=True)
    
    17
    +    with open(path, 'w') as f:
    
    18
    +        f.write(content)
    
    19
    +        os.fchmod(f.fileno(), mode)
    
    20
    +
    
    21
    +
    
    22
    +def create_test_directory(*path, mode=0o644):
    
    23
    +    create_test_file(*path, '.keep', content='')
    
    24
    +    path = os.path.join(*path)
    
    25
    +    os.chmod(path, mode)
    
    26
    +
    
    27
    +
    
    28
    +@pytest.mark.integration
    
    29
    +@pytest.mark.datafiles(DATA_DIR)
    
    30
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    
    31
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    
    32
    +    project = str(datafiles)
    
    33
    +    element_name = 'list'
    
    34
    +    element_path = os.path.join(project, 'elements', element_name)
    
    35
    +    repodir = os.path.join(str(tmpdir), 'repo')
    
    36
    +    sourcedir = os.path.join(project, 'source')
    
    37
    +
    
    38
    +    create_test_file(sourcedir, 'a.txt', mode=0o700)
    
    39
    +    create_test_file(sourcedir, 'b.txt', mode=0o755)
    
    40
    +    create_test_file(sourcedir, 'c.txt', mode=0o600)
    
    41
    +    create_test_file(sourcedir, 'd.txt', mode=0o400)
    
    42
    +    create_test_file(sourcedir, 'e.txt', mode=0o644)
    
    43
    +    create_test_file(sourcedir, 'f.txt', mode=0o4755)
    
    44
    +    create_test_file(sourcedir, 'g.txt', mode=0o2755)
    
    45
    +    create_test_file(sourcedir, 'h.txt', mode=0o1755)
    
    46
    +    create_test_directory(sourcedir, 'dir-a', mode=0o0700)
    
    47
    +    create_test_directory(sourcedir, 'dir-c', mode=0o0755)
    
    48
    +    create_test_directory(sourcedir, 'dir-d', mode=0o4755)
    
    49
    +    create_test_directory(sourcedir, 'dir-e', mode=0o2755)
    
    50
    +    create_test_directory(sourcedir, 'dir-f', mode=0o1755)
    
    51
    +
    
    52
    +    if kind == 'local':
    
    53
    +        source = {'kind': 'local',
    
    54
    +                  'path': 'source'}
    
    55
    +    else:
    
    56
    +        repo = create_repo(kind, repodir)
    
    57
    +        ref = repo.create(sourcedir)
    
    58
    +        source = repo.source_config(ref=ref)
    
    59
    +    element = {
    
    60
    +        'kind': 'manual',
    
    61
    +        'depends': [
    
    62
    +            {
    
    63
    +                'filename': 'base.bst',
    
    64
    +                'type': 'build'
    
    65
    +            }
    
    66
    +        ],
    
    67
    +        'sources': [
    
    68
    +            source
    
    69
    +        ],
    
    70
    +        'config': {
    
    71
    +            'install-commands': [
    
    72
    +                'ls -l >"%{install-root}/ls-l"'
    
    73
    +            ]
    
    74
    +        }
    
    75
    +    }
    
    76
    +    _yaml.dump(element, element_path)
    
    77
    +
    
    78
    +    def get_value_for_umask(umask):
    
    79
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
    
    80
    +
    
    81
    +        old_umask = os.umask(umask)
    
    82
    +
    
    83
    +        try:
    
    84
    +            result = cli.run(project=project, args=['build', element_name])
    
    85
    +            result.assert_success()
    
    86
    +
    
    87
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    
    88
    +            result.assert_success()
    
    89
    +
    
    90
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    91
    +                return f.read()
    
    92
    +        finally:
    
    93
    +            os.umask(old_umask)
    
    94
    +            cli.remove_artifact_from_cache(project, element_name)
    
    95
    +
    
    96
    +    assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    
    97
    +
    
    98
    +
    
    99
    +@pytest.mark.integration
    
    100
    +@pytest.mark.datafiles(DATA_DIR)
    
    101
    +def test_deterministic_source_local(cli, tmpdir, datafiles):
    
    102
    +    """Only user rights should be considered for local source.
    
    103
    +    """
    
    104
    +    project = str(datafiles)
    
    105
    +    element_name = 'test'
    
    106
    +    element_path = os.path.join(project, 'elements', element_name)
    
    107
    +    sourcedir = os.path.join(project, 'source')
    
    108
    +
    
    109
    +    element = {
    
    110
    +        'kind': 'manual',
    
    111
    +        'depends': [
    
    112
    +            {
    
    113
    +                'filename': 'base.bst',
    
    114
    +                'type': 'build'
    
    115
    +            }
    
    116
    +        ],
    
    117
    +        'sources': [
    
    118
    +            {
    
    119
    +                'kind': 'local',
    
    120
    +                'path': 'source'
    
    121
    +            }
    
    122
    +        ],
    
    123
    +        'config': {
    
    124
    +            'install-commands': [
    
    125
    +                'ls -l >"%{install-root}/ls-l"'
    
    126
    +            ]
    
    127
    +        }
    
    128
    +    }
    
    129
    +    _yaml.dump(element, element_path)
    
    130
    +
    
    131
    +    def get_value_for_mask(mask):
    
    132
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
    
    133
    +
    
    134
    +        create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
    
    135
    +        create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
    
    136
    +        create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
    
    137
    +        create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
    
    138
    +        create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
    
    139
    +        create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
    
    140
    +        create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
    
    141
    +        create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
    
    142
    +        create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
    
    143
    +        try:
    
    144
    +            result = cli.run(project=project, args=['build', element_name])
    
    145
    +            result.assert_success()
    
    146
    +
    
    147
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    
    148
    +            result.assert_success()
    
    149
    +
    
    150
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    151
    +                return f.read()
    
    152
    +        finally:
    
    153
    +            cli.remove_artifact_from_cache(project, element_name)
    
    154
    +
    
    155
    +    assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]