GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/deterministic-source] 4 commits: Use deterministic umask when staging sources.

Title: GitLab

Valentin David pushed to branch valentindavid/deterministic-source at BuildStream / buildstream

Commits:

4 changed files:

Changes:

  • buildstream/plugins/sources/remote.py
    ... ... @@ -49,6 +49,7 @@ remote - stage files from remote urls
    49 49
    50 50 
     """
    51 51 
     import os
    52 
    +import stat
    52 53 
     from buildstream import SourceError, utils
    53 54 
     from ._downloadablefilesource import DownloadableFileSource
    54 55
    ... ... @@ -75,6 +76,7 @@ class RemoteSource(DownloadableFileSource):
    75 76 
             dest = os.path.join(directory, self.filename)
    76 77 
             with self.timed_activity("Staging remote file to {}".format(dest)):
    77 78 
                 utils.safe_copy(self._get_mirror_file(), dest)
    79 
    +            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    78 80
    79 81
    80 82 
     def setup():

  • buildstream/plugins/sources/zip.py
    ... ... @@ -49,10 +49,17 @@ zip - stage files from zip archives
    49 49 
        # To extract the root of the archive directly, this can be set
    50 50 
        # to an empty string.
    51 51 
        base-dir: '*'
    52 
    +
    53 
    +.. attention::
    54 
    +
    55 
    +   File permissions are not preserved. All extracted directories have
    56 
    +   permissions 0755 and all extracted files have permissions 0644.
    57 
    +
    52 58 
     """
    53 59
    54 60 
     import os
    55 61 
     import zipfile
    62 
    +import stat
    56 63
    57 64 
     from buildstream import SourceError
    58 65 
     from buildstream import utils
    ... ... @@ -74,6 +81,9 @@ class ZipSource(DownloadableFileSource):
    74 81 
             return super().get_unique_key() + [self.base_dir]
    75 82
    76 83 
         def stage(self, directory):
    84 
    +        exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
    85 
    +        noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    86 
    +
    77 87 
             try:
    78 88 
                 with zipfile.ZipFile(self._get_mirror_file()) as archive:
    79 89 
                     base_dir = None
    ... ... @@ -81,9 +91,27 @@ class ZipSource(DownloadableFileSource):
    81 91 
                         base_dir = self._find_base_dir(archive, self.base_dir)
    82 92
    83 93 
                     if base_dir:
    84 
    -                    archive.extractall(path=directory, members=self._extract_members(archive, base_dir))
    94 
    +                    members = self._extract_members(archive, base_dir)
    85 95 
                     else:
    86 
    -                    archive.extractall(path=directory)
    96 
    +                    members = archive.namelist()
    97 
    +
    98 
    +                for member in members:
    99 
    +                    written = archive.extract(member, path=directory)
    100 
    +
    101 
    +                    # zipfile.extract might create missing directories
    102 
    +                    rel = os.path.relpath(written, start=directory)
    103 
    +                    assert not os.path.isabs(rel)
    104 
    +                    rel = os.path.dirname(rel)
    105 
    +                    while rel:
    106 
    +                        os.chmod(os.path.join(directory, rel), exec_rights)
    107 
    +                        rel = os.path.dirname(rel)
    108 
    +
    109 
    +                    if os.path.islink(written):
    110 
    +                        pass
    111 
    +                    elif os.path.isdir(written):
    112 
    +                        os.chmod(written, exec_rights)
    113 
    +                    else:
    114 
    +                        os.chmod(written, noexec_rights)
    87 115
    88 116 
             except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
    89 117 
                 raise SourceError("{}: Error staging source: {}".format(self, e)) from e

  • buildstream/utils.py
    ... ... @@ -1010,6 +1010,15 @@ def _call(*popenargs, terminate=False, **kwargs):
    1010 1010
    1011 1011 
         process = None
    1012 1012
    1013 
    +    old_preexec_fn = kwargs.get('preexec_fn')
    1014 
    +    if 'preexec_fn' in kwargs:
    1015 
    +        del kwargs['preexec_fn']
    1016 
    +
    1017 
    +    def preexec_fn():
    1018 
    +        os.umask(stat.S_IWGRP | stat.S_IWOTH)
    1019 
    +        if old_preexec_fn is not None:
    1020 
    +            old_preexec_fn()
    1021 
    +
    1013 1022 
         # Handle termination, suspend and resume
    1014 1023 
         def kill_proc():
    1015 1024 
             if process:
    ... ... @@ -1054,7 +1063,7 @@ def _call(*popenargs, terminate=False, **kwargs):
    1054 1063 
                 os.killpg(group_id, signal.SIGCONT)
    1055 1064
    1056 1065 
         with _signals.suspendable(suspend_proc, resume_proc), _signals.terminator(kill_proc):
    1057 
    -        process = subprocess.Popen(*popenargs, **kwargs)
    1066 
    +        process = subprocess.Popen(*popenargs, preexec_fn=preexec_fn, **kwargs)
    1058 1067 
             output, _ = process.communicate()
    1059 1068 
             exit_code = process.poll()
    1060 1069

  • tests/integration/source-determinism.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream import _yaml, utils
    5 
    +from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    6 
    +
    7 
    +
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project"
    11 
    +)
    12 
    +
    13 
    +
    14 
    +def create_test_file(*path, mode=0o644, content='content\n'):
    15 
    +    path = os.path.join(*path)
    16 
    +    os.makedirs(os.path.dirname(path), exist_ok=True)
    17 
    +    with open(path, 'w') as f:
    18 
    +        f.write(content)
    19 
    +        os.fchmod(f.fileno(), mode)
    20 
    +
    21 
    +
    22 
    +def create_test_directory(*path, mode=0o644):
    23 
    +    create_test_file(*path, '.keep', content='')
    24 
    +    path = os.path.join(*path)
    25 
    +    os.chmod(path, mode)
    26 
    +
    27 
    +
    28 
    +@pytest.mark.integration
    29 
    +@pytest.mark.datafiles(DATA_DIR)
    30 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    31 
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    32 
    +    project = str(datafiles)
    33 
    +    element_name = 'list'
    34 
    +    element_path = os.path.join(project, 'elements', element_name)
    35 
    +    repodir = os.path.join(str(tmpdir), 'repo')
    36 
    +    sourcedir = os.path.join(project, 'source')
    37 
    +
    38 
    +    create_test_file(sourcedir, 'a.txt', mode=0o700)
    39 
    +    create_test_file(sourcedir, 'b.txt', mode=0o755)
    40 
    +    create_test_file(sourcedir, 'c.txt', mode=0o600)
    41 
    +    create_test_file(sourcedir, 'd.txt', mode=0o400)
    42 
    +    create_test_file(sourcedir, 'e.txt', mode=0o644)
    43 
    +    create_test_file(sourcedir, 'f.txt', mode=0o4755)
    44 
    +    create_test_file(sourcedir, 'g.txt', mode=0o2755)
    45 
    +    create_test_file(sourcedir, 'h.txt', mode=0o1755)
    46 
    +    create_test_directory(sourcedir, 'dir-a', mode=0o0700)
    47 
    +    create_test_directory(sourcedir, 'dir-c', mode=0o0755)
    48 
    +    create_test_directory(sourcedir, 'dir-d', mode=0o4755)
    49 
    +    create_test_directory(sourcedir, 'dir-e', mode=0o2755)
    50 
    +    create_test_directory(sourcedir, 'dir-f', mode=0o1755)
    51 
    +
    52 
    +    if kind == 'local':
    53 
    +        source = {'kind': 'local',
    54 
    +                  'path': 'source'}
    55 
    +    else:
    56 
    +        repo = create_repo(kind, repodir)
    57 
    +        ref = repo.create(sourcedir)
    58 
    +        source = repo.source_config(ref=ref)
    59 
    +    element = {
    60 
    +        'kind': 'manual',
    61 
    +        'depends': [
    62 
    +            {
    63 
    +                'filename': 'base.bst',
    64 
    +                'type': 'build'
    65 
    +            }
    66 
    +        ],
    67 
    +        'sources': [
    68 
    +            source
    69 
    +        ],
    70 
    +        'config': {
    71 
    +            'install-commands': [
    72 
    +                'ls -l >"%{install-root}/ls-l"'
    73 
    +            ]
    74 
    +        }
    75 
    +    }
    76 
    +    _yaml.dump(element, element_path)
    77 
    +
    78 
    +    def get_value_for_umask(umask):
    79 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
    80 
    +
    81 
    +        old_umask = os.umask(umask)
    82 
    +
    83 
    +        try:
    84 
    +            result = cli.run(project=project, args=['build', element_name])
    85 
    +            result.assert_success()
    86 
    +
    87 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    88 
    +            result.assert_success()
    89 
    +
    90 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    91 
    +                return f.read()
    92 
    +        finally:
    93 
    +            os.umask(old_umask)
    94 
    +            cli.remove_artifact_from_cache(project, element_name)
    95 
    +
    96 
    +    assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    97 
    +
    98 
    +
    99 
    +@pytest.mark.integration
    100 
    +@pytest.mark.datafiles(DATA_DIR)
    101 
    +def test_deterministic_source_local(cli, tmpdir, datafiles):
    102 
    +    """Only user rights should be considered for local source.
    103 
    +    """
    104 
    +    project = str(datafiles)
    105 
    +    element_name = 'test'
    106 
    +    element_path = os.path.join(project, 'elements', element_name)
    107 
    +    sourcedir = os.path.join(project, 'source')
    108 
    +
    109 
    +    element = {
    110 
    +        'kind': 'manual',
    111 
    +        'depends': [
    112 
    +            {
    113 
    +                'filename': 'base.bst',
    114 
    +                'type': 'build'
    115 
    +            }
    116 
    +        ],
    117 
    +        'sources': [
    118 
    +            {
    119 
    +                'kind': 'local',
    120 
    +                'path': 'source'
    121 
    +            }
    122 
    +        ],
    123 
    +        'config': {
    124 
    +            'install-commands': [
    125 
    +                'ls -l >"%{install-root}/ls-l"'
    126 
    +            ]
    127 
    +        }
    128 
    +    }
    129 
    +    _yaml.dump(element, element_path)
    130 
    +
    131 
    +    def get_value_for_mask(mask):
    132 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
    133 
    +
    134 
    +        create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
    135 
    +        create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
    136 
    +        create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
    137 
    +        create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
    138 
    +        create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
    139 
    +        create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
    140 
    +        create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
    141 
    +        create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
    142 
    +        create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
    143 
    +        try:
    144 
    +            result = cli.run(project=project, args=['build', element_name])
    145 
    +            result.assert_success()
    146 
    +
    147 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    148 
    +            result.assert_success()
    149 
    +
    150 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    151 
    +                return f.read()
    152 
    +        finally:
    153 
    +            cli.remove_artifact_from_cache(project, element_name)
    154 
    +
    155 
    +    assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]