[Notes] [Git][BuildStream/buildstream][valentindavid/deterministic-source] 4 commits: Use deterministic umask when staging sources.



Title: GitLab

Valentin David pushed to branch valentindavid/deterministic-source at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/plugins/sources/remote.py
    ... ... @@ -49,6 +49,7 @@ remote - stage files from remote urls
    49 49
     
    
    50 50
     """
    
    51 51
     import os
    
    52
    +import stat
    
    52 53
     from buildstream import SourceError, utils
    
    53 54
     from ._downloadablefilesource import DownloadableFileSource
    
    54 55
     
    
    ... ... @@ -75,6 +76,7 @@ class RemoteSource(DownloadableFileSource):
    75 76
             dest = os.path.join(directory, self.filename)
    
    76 77
             with self.timed_activity("Staging remote file to {}".format(dest)):
    
    77 78
                 utils.safe_copy(self._get_mirror_file(), dest)
    
    79
    +            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    
    78 80
     
    
    79 81
     
    
    80 82
     def setup():
    

  • buildstream/plugins/sources/zip.py
    ... ... @@ -49,10 +49,17 @@ zip - stage files from zip archives
    49 49
        # To extract the root of the archive directly, this can be set
    
    50 50
        # to an empty string.
    
    51 51
        base-dir: '*'
    
    52
    +
    
    53
    +.. attention::
    
    54
    +
    
    55
    +   File permissions are not preserved. All extracted directories have
    
    56
    +   permissions 0755 and all extracted files have permissions 0644.
    
    57
    +
    
    52 58
     """
    
    53 59
     
    
    54 60
     import os
    
    55 61
     import zipfile
    
    62
    +import stat
    
    56 63
     
    
    57 64
     from buildstream import SourceError
    
    58 65
     from buildstream import utils
    
    ... ... @@ -74,6 +81,9 @@ class ZipSource(DownloadableFileSource):
    74 81
             return super().get_unique_key() + [self.base_dir]
    
    75 82
     
    
    76 83
         def stage(self, directory):
    
    84
    +        exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
    
    85
    +        noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    
    86
    +
    
    77 87
             try:
    
    78 88
                 with zipfile.ZipFile(self._get_mirror_file()) as archive:
    
    79 89
                     base_dir = None
    
    ... ... @@ -81,9 +91,27 @@ class ZipSource(DownloadableFileSource):
    81 91
                         base_dir = self._find_base_dir(archive, self.base_dir)
    
    82 92
     
    
    83 93
                     if base_dir:
    
    84
    -                    archive.extractall(path=directory, members=self._extract_members(archive, base_dir))
    
    94
    +                    members = self._extract_members(archive, base_dir)
    
    85 95
                     else:
    
    86
    -                    archive.extractall(path=directory)
    
    96
    +                    members = archive.namelist()
    
    97
    +
    
    98
    +                for member in members:
    
    99
    +                    written = archive.extract(member, path=directory)
    
    100
    +
    
    101
    +                    # zipfile.extract might create missing directories
    
    102
    +                    rel = os.path.relpath(written, start=directory)
    
    103
    +                    assert not os.path.isabs(rel)
    
    104
    +                    rel = os.path.dirname(rel)
    
    105
    +                    while rel:
    
    106
    +                        os.chmod(os.path.join(directory, rel), exec_rights)
    
    107
    +                        rel = os.path.dirname(rel)
    
    108
    +
    
    109
    +                    if os.path.islink(written):
    
    110
    +                        pass
    
    111
    +                    elif os.path.isdir(written):
    
    112
    +                        os.chmod(written, exec_rights)
    
    113
    +                    else:
    
    114
    +                        os.chmod(written, noexec_rights)
    
    87 115
     
    
    88 116
             except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
    
    89 117
                 raise SourceError("{}: Error staging source: {}".format(self, e)) from e
    

  • buildstream/utils.py
    ... ... @@ -1010,6 +1010,15 @@ def _call(*popenargs, terminate=False, **kwargs):
    1010 1010
     
    
    1011 1011
         process = None
    
    1012 1012
     
    
    1013
    +    old_preexec_fn = kwargs.get('preexec_fn')
    
    1014
    +    if 'preexec_fn' in kwargs:
    
    1015
    +        del kwargs['preexec_fn']
    
    1016
    +
    
    1017
    +    def preexec_fn():
    
    1018
    +        os.umask(stat.S_IWGRP | stat.S_IWOTH)
    
    1019
    +        if old_preexec_fn is not None:
    
    1020
    +            old_preexec_fn()
    
    1021
    +
    
    1013 1022
         # Handle termination, suspend and resume
    
    1014 1023
         def kill_proc():
    
    1015 1024
             if process:
    
    ... ... @@ -1054,7 +1063,7 @@ def _call(*popenargs, terminate=False, **kwargs):
    1054 1063
                 os.killpg(group_id, signal.SIGCONT)
    
    1055 1064
     
    
    1056 1065
         with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
    
    1057
    -        process = subprocess.Popen(*popenargs, **kwargs)
    
    1066
    +        process = subprocess.Popen(*popenargs, preexec_fn=preexec_fn, **kwargs)
    
    1058 1067
             output, _ = process.communicate()
    
    1059 1068
             exit_code = process.poll()
    
    1060 1069
     
    

  • tests/integration/source-determinism.py
    1
    +import os
    
    2
    +import pytest
    
    3
    +
    
    4
    +from buildstream import _yaml, utils
    
    5
    +from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    
    6
    +
    
    7
    +
    
    8
    +DATA_DIR = os.path.join(
    
    9
    +    os.path.dirname(os.path.realpath(__file__)),
    
    10
    +    "project"
    
    11
    +)
    
    12
    +
    
    13
    +
    
    14
    +def create_test_file(*path, mode=0o644, content='content\n'):
    
    15
    +    path = os.path.join(*path)
    
    16
    +    os.makedirs(os.path.dirname(path), exist_ok=True)
    
    17
    +    with open(path, 'w') as f:
    
    18
    +        f.write(content)
    
    19
    +        os.fchmod(f.fileno(), mode)
    
    20
    +
    
    21
    +
    
    22
    +def create_test_directory(*path, mode=0o644):
    
    23
    +    create_test_file(*path, '.keep', content='')
    
    24
    +    path = os.path.join(*path)
    
    25
    +    os.chmod(path, mode)
    
    26
    +
    
    27
    +
    
    28
    +@pytest.mark.integration
    
    29
    +@pytest.mark.datafiles(DATA_DIR)
    
    30
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    
    31
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    
    32
    +    project = str(datafiles)
    
    33
    +    element_name = 'list'
    
    34
    +    element_path = os.path.join(project, 'elements', element_name)
    
    35
    +    repodir = os.path.join(str(tmpdir), 'repo')
    
    36
    +    sourcedir = os.path.join(project, 'source')
    
    37
    +
    
    38
    +    create_test_file(sourcedir, 'a.txt', mode=0o700)
    
    39
    +    create_test_file(sourcedir, 'b.txt', mode=0o755)
    
    40
    +    create_test_file(sourcedir, 'c.txt', mode=0o600)
    
    41
    +    create_test_file(sourcedir, 'd.txt', mode=0o400)
    
    42
    +    create_test_file(sourcedir, 'e.txt', mode=0o644)
    
    43
    +    create_test_file(sourcedir, 'f.txt', mode=0o4755)
    
    44
    +    create_test_file(sourcedir, 'g.txt', mode=0o2755)
    
    45
    +    create_test_file(sourcedir, 'h.txt', mode=0o1755)
    
    46
    +    create_test_directory(sourcedir, 'dir-a', mode=0o0700)
    
    47
    +    create_test_directory(sourcedir, 'dir-c', mode=0o0755)
    
    48
    +    create_test_directory(sourcedir, 'dir-d', mode=0o4755)
    
    49
    +    create_test_directory(sourcedir, 'dir-e', mode=0o2755)
    
    50
    +    create_test_directory(sourcedir, 'dir-f', mode=0o1755)
    
    51
    +
    
    52
    +    if kind == 'local':
    
    53
    +        source = {'kind': 'local',
    
    54
    +                  'path': 'source'}
    
    55
    +    else:
    
    56
    +        repo = create_repo(kind, repodir)
    
    57
    +        ref = repo.create(sourcedir)
    
    58
    +        source = repo.source_config(ref=ref)
    
    59
    +    element = {
    
    60
    +        'kind': 'manual',
    
    61
    +        'depends': [
    
    62
    +            {
    
    63
    +                'filename': 'base.bst',
    
    64
    +                'type': 'build'
    
    65
    +            }
    
    66
    +        ],
    
    67
    +        'sources': [
    
    68
    +            source
    
    69
    +        ],
    
    70
    +        'config': {
    
    71
    +            'install-commands': [
    
    72
    +                'ls -l >"%{install-root}/ls-l"'
    
    73
    +            ]
    
    74
    +        }
    
    75
    +    }
    
    76
    +    _yaml.dump(element, element_path)
    
    77
    +
    
    78
    +    def get_value_for_umask(umask):
    
    79
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
    
    80
    +
    
    81
    +        old_umask = os.umask(umask)
    
    82
    +
    
    83
    +        try:
    
    84
    +            result = cli.run(project=project, args=['build', element_name])
    
    85
    +            result.assert_success()
    
    86
    +
    
    87
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    
    88
    +            result.assert_success()
    
    89
    +
    
    90
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    91
    +                return f.read()
    
    92
    +        finally:
    
    93
    +            os.umask(old_umask)
    
    94
    +            cli.remove_artifact_from_cache(project, element_name)
    
    95
    +
    
    96
    +    assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    
    97
    +
    
    98
    +
    
    99
    +@pytest.mark.integration
    
    100
    +@pytest.mark.datafiles(DATA_DIR)
    
    101
    +def test_deterministic_source_local(cli, tmpdir, datafiles):
    
    102
    +    """Only user rights should be considered for local source.
    
    103
    +    """
    
    104
    +    project = str(datafiles)
    
    105
    +    element_name = 'test'
    
    106
    +    element_path = os.path.join(project, 'elements', element_name)
    
    107
    +    sourcedir = os.path.join(project, 'source')
    
    108
    +
    
    109
    +    element = {
    
    110
    +        'kind': 'manual',
    
    111
    +        'depends': [
    
    112
    +            {
    
    113
    +                'filename': 'base.bst',
    
    114
    +                'type': 'build'
    
    115
    +            }
    
    116
    +        ],
    
    117
    +        'sources': [
    
    118
    +            {
    
    119
    +                'kind': 'local',
    
    120
    +                'path': 'source'
    
    121
    +            }
    
    122
    +        ],
    
    123
    +        'config': {
    
    124
    +            'install-commands': [
    
    125
    +                'ls -l >"%{install-root}/ls-l"'
    
    126
    +            ]
    
    127
    +        }
    
    128
    +    }
    
    129
    +    _yaml.dump(element, element_path)
    
    130
    +
    
    131
    +    def get_value_for_mask(mask):
    
    132
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
    
    133
    +
    
    134
    +        create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
    
    135
    +        create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
    
    136
    +        create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
    
    137
    +        create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
    
    138
    +        create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
    
    139
    +        create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
    
    140
    +        create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
    
    141
    +        create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
    
    142
    +        create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
    
    143
    +        try:
    
    144
    +            result = cli.run(project=project, args=['build', element_name])
    
    145
    +            result.assert_success()
    
    146
    +
    
    147
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    
    148
    +            result.assert_success()
    
    149
    +
    
    150
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    
    151
    +                return f.read()
    
    152
    +        finally:
    
    153
    +            cli.remove_artifact_from_cache(project, element_name)
    
    154
    +
    
    155
    +    assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)



  • [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]