GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/deterministic-source] 6 commits: buildstream/plugins/sources/local.py: Make staging deterministic.

Title: GitLab

Valentin David pushed to branch valentindavid/deterministic-source at BuildStream / buildstream

Commits:

9 changed files:

Changes:

  • buildstream/plugins/sources/bzr.py
    ... ... @@ -123,7 +123,8 @@ class BzrSource(Source):
    123 123 
                        "--revision=revno:{}".format(self.ref),
    124 124 
                        self._get_branch_dir(), directory],
    125 125 
                       fail="Failed to checkout revision {} from branch {} to {}"
    126 
    -                  .format(self.ref, self._get_branch_dir(), directory))
    126 
    +                  .format(self.ref, self._get_branch_dir(), directory),
    127 
    +                  deterministic_umask=True)
    127 128 
             # Remove .bzr dir
    128 129 
             shutil.rmtree(os.path.join(directory, ".bzr"))
    129 130

  • buildstream/plugins/sources/git.py
    ... ... @@ -198,11 +198,13 @@ class GitMirror(SourceFetcher):
    198 198 
             # directory.
    199 199 
             self.source.call([self.source.host_git, 'clone', '--no-checkout', '--shared', self.mirror, fullpath],
    200 200 
                              fail="Failed to create git mirror {} in directory: {}".format(self.mirror, fullpath),
    201 
    -                         fail_temporarily=True)
    201 
    +                         fail_temporarily=True,
    202 
    +                         deterministic_umask=True)
    202 203
    203 204 
             self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
    204 205 
                              fail="Failed to checkout git ref {}".format(self.ref),
    205 
    -                         cwd=fullpath)
    206 
    +                         cwd=fullpath,
    207 
    +                         deterministic_umask=True)
    206 208
    207 209 
             # Remove .git dir
    208 210 
             shutil.rmtree(os.path.join(fullpath, ".git"))

  • buildstream/plugins/sources/local.py
    ... ... @@ -34,6 +34,10 @@ local - stage local files and directories
    34 34
    35 35 
        # Specify the project relative path to a file or directory
    36 36 
        path: files/somefile.txt
    37 
    +
    38 
    +   # Specifiy whether we want deterministic staging.
    39 
    +   # Setting to False would copy all metadata to the staged files.
    40 
    +   deterministic: True
    37 41 
     """
    38 42
    39 43 
     import os
    ... ... @@ -51,8 +55,9 @@ class LocalSource(Source):
    51 55 
             self.__unique_key = None
    52 56
    53 57 
         def configure(self, node):
    54 
    -        self.node_validate(node, ['path'] + Source.COMMON_CONFIG_KEYS)
    58 
    +        self.node_validate(node, ['path', 'deterministic'] + Source.COMMON_CONFIG_KEYS)
    55 59 
             self.path = self.node_get_project_path(node, 'path')
    60 
    +        self.deterministic = self.node_get_member(node, bool, 'deterministic', True)
    56 61 
             self.fullpath = os.path.join(self.get_project_directory(), self.path)
    57 62
    58 63 
         def preflight(self):
    ... ... @@ -95,10 +100,10 @@ class LocalSource(Source):
    95 100 
             # in the sandbox.
    96 101 
             with self.timed_activity("Staging local files at {}".format(self.path)):
    97 102 
                 if os.path.isdir(self.fullpath):
    98 
    -                utils.copy_files(self.fullpath, directory)
    103 
    +                utils.copy_files(self.fullpath, directory, deterministic=self.deterministic)
    99 104 
                 else:
    100 105 
                     destfile = os.path.join(directory, os.path.basename(self.path))
    101 
    -                utils.safe_copy(self.fullpath, destfile)
    106 
    +                utils.safe_copy(self.fullpath, destfile, deterministic=self.deterministic)
    102 107
    103 108
    104 109 
     # Create a unique key for a file

  • buildstream/plugins/sources/patch.py
    ... ... @@ -90,9 +90,10 @@ class PatchSource(Source):
    90 90 
                     raise SourceError("Nothing to patch in directory '{}'".format(directory),
    91 91 
                                       reason="patch-no-files")
    92 92
    93 
    -            strip_level_option = "-p{}".format(self.strip_level)
    94 
    -            self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
    95 
    -                      fail="Failed to apply patch {}".format(self.path))
    93 
    +        strip_level_option = "-p{}".format(self.strip_level)
    94 
    +        self.call([self.host_patch, strip_level_option, "-i", self.fullpath, "-d", directory],
    95 
    +                  fail="Failed to apply patch {}".format(self.path),
    96 
    +                  deterministic_umask=True)
    96 97
    97 98
    98 99 
     # Plugin entry point

  • buildstream/plugins/sources/remote.py
    ... ... @@ -74,7 +74,7 @@ class RemoteSource(DownloadableFileSource):
    74 74 
             # are not write protected in the sandbox.
    75 75 
             dest = os.path.join(directory, self.filename)
    76 76 
             with self.timed_activity("Staging remote file to {}".format(dest)):
    77 
    -            utils.safe_copy(self._get_mirror_file(), dest)
    77 
    +            utils.safe_copy(self._get_mirror_file(), dest, deterministic=True)
    78 78
    79 79
    80 80 
     def setup():

  • buildstream/plugins/sources/tar.py
    ... ... @@ -115,9 +115,12 @@ class TarSource(DownloadableFileSource):
    115 115 
                         base_dir = self._find_base_dir(tar, self.base_dir)
    116 116
    117 117 
                     if base_dir:
    118 
    -                    tar.extractall(path=directory, members=self._extract_members(tar, base_dir))
    118 
    +                    members = self._extract_members(tar, base_dir)
    119 
    +                    tar.extractall(path=directory,
    120 
    +                                   members=_deterministic_user(members))
    119 121 
                     else:
    120 
    -                    tar.extractall(path=directory)
    122 
    +                    tar.extractall(path=directory,
    123 
    +                                   members=_deterministic_user(tar))
    121 124
    122 125 
             except (tarfile.TarError, OSError) as e:
    123 126 
                 raise SourceError("{}: Error staging source: {}".format(self, e)) from e
    ... ... @@ -199,5 +202,15 @@ class TarSource(DownloadableFileSource):
    199 202 
             return matches[0]
    200 203
    201 204
    205 
    +# Set deterministic user in metadata
    206 
    +def _deterministic_user(tarinfos):
    207 
    +    for tarinfo in tarinfos:
    208 
    +        tarinfo.uid = 0
    209 
    +        tarinfo.gid = 0
    210 
    +        tarinfo.uname = 'root'
    211 
    +        tarinfo.gname = 'root'
    212 
    +        yield tarinfo
    213 
    +
    214 
    +
    202 215 
     def setup():
    203 216 
         return TarSource

  • buildstream/plugins/sources/zip.py
    ... ... @@ -53,6 +53,7 @@ zip - stage files from zip archives
    53 53
    54 54 
     import os
    55 55 
     import zipfile
    56 
    +import stat
    56 57
    57 58 
     from buildstream import SourceError
    58 59 
     from buildstream import utils
    ... ... @@ -74,6 +75,9 @@ class ZipSource(DownloadableFileSource):
    74 75 
             return super().get_unique_key() + [self.base_dir]
    75 76
    76 77 
         def stage(self, directory):
    78 
    +        exec_rights = (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH)
    79 
    +        noexec_rights = exec_rights & ~(stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
    80 
    +
    77 81 
             try:
    78 82 
                 with zipfile.ZipFile(self._get_mirror_file()) as archive:
    79 83 
                     base_dir = None
    ... ... @@ -81,9 +85,27 @@ class ZipSource(DownloadableFileSource):
    81 85 
                         base_dir = self._find_base_dir(archive, self.base_dir)
    82 86
    83 87 
                     if base_dir:
    84 
    -                    archive.extractall(path=directory, members=self._extract_members(archive, base_dir))
    88 
    +                    members = self._extract_members(archive, base_dir)
    85 89 
                     else:
    86 
    -                    archive.extractall(path=directory)
    90 
    +                    members = archive.namelist()
    91 
    +
    92 
    +                for member in members:
    93 
    +                    written = archive.extract(member, path=directory)
    94 
    +
    95 
    +                    # zipfile.extract might create missing directories
    96 
    +                    rel = os.path.relpath(written, start=directory)
    97 
    +                    assert not os.path.isabs(rel)
    98 
    +                    rel = os.path.dirname(rel)
    99 
    +                    while rel:
    100 
    +                        os.chmod(os.path.join(directory, rel), exec_rights)
    101 
    +                        rel = os.path.dirname(rel)
    102 
    +
    103 
    +                    if os.path.islink(written):
    104 
    +                        pass
    105 
    +                    elif os.path.isdir(written):
    106 
    +                        os.chmod(written, exec_rights)
    107 
    +                    else:
    108 
    +                        os.chmod(written, noexec_rights)
    87 109
    88 110 
             except (zipfile.BadZipFile, zipfile.LargeZipFile, OSError) as e:
    89 111 
                 raise SourceError("{}: Error staging source: {}".format(self, e)) from e

  • buildstream/utils.py
    ... ... @@ -239,12 +239,15 @@ def sha256sum(filename):
    239 239 
         return h.hexdigest()
    240 240
    241 241
    242 
    -def safe_copy(src, dest, *, result=None):
    243 
    -    """Copy a file while preserving attributes
    242 
    +def safe_copy(src, dest, *, deterministic=False, result=None):
    243 
    +    """Copy a file while preserving attributes.
    244 
    +
    245 
    +    In deterministic mode, only execution rights are preserved.
    244 246
    245 247 
         Args:
    246 248 
            src (str): The source filename
    247 249 
            dest (str): The destination filename
    250 
    +       deterministic (bool): Whether to care only about execution rights
    248 251 
            result (:class:`~.FileListResult`): An optional collective result
    249 252
    250 253 
         Raises:
    ... ... @@ -263,6 +266,13 @@ def safe_copy(src, dest, *, result=None):
    263 266 
                                 .format(dest, e)) from e
    264 267
    265 268 
         shutil.copyfile(src, dest)
    269 
    +    if deterministic:
    270 
    +        st = os.stat(src)
    271 
    +        if st.st_mode & stat.S_IXUSR:
    272 
    +            os.chmod(dest, stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH)
    273 
    +        else:
    274 
    +            os.chmod(dest, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
    275 
    +        return
    266 276 
         try:
    267 277 
             shutil.copystat(src, dest)
    268 278 
         except PermissionError:
    ... ... @@ -280,7 +290,7 @@ def safe_copy(src, dest, *, result=None):
    280 290 
                             .format(src, dest, e)) from e
    281 291
    282 292
    283 
    -def safe_link(src, dest, *, result=None):
    293 
    +def safe_link(src, dest, *, result=None, **kwargs):
    284 294 
         """Try to create a hardlink, but resort to copying in the case of cross device links.
    285 295
    286 296 
         Args:
    ... ... @@ -350,7 +360,8 @@ def safe_remove(path):
    350 360 
         return True
    351 361
    352 362
    353 
    -def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False):
    363 
    +def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=False,
    364 
    +               deterministic=False):
    354 365 
         """Copy files from source to destination.
    355 366
    356 367 
         Args:
    ... ... @@ -359,6 +370,8 @@ def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
    359 370 
            files (list): Optional list of files in `src` to copy
    360 371 
            ignore_missing (bool): Dont raise any error if a source file is missing
    361 372 
            report_written (bool): Add to the result object the full list of files written
    373 
    +       deterministic (bool): Whether to care only about execution rights
    374 
    +
    362 375
    363 376 
         Returns:
    364 377 
            (:class:`~.FileListResult`): The result describing what happened during this file operation
    ... ... @@ -380,7 +393,8 @@ def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
    380 393 
         result = FileListResult()
    381 394 
         try:
    382 395 
             _process_list(src, dest, files, safe_copy, result, ignore_missing=ignore_missing,
    383 
    -                      report_written=report_written, presorted=presorted)
    396 
    +                      report_written=report_written, presorted=presorted,
    397 
    +                      deterministic=deterministic)
    384 398 
         except OSError as e:
    385 399 
             raise UtilError("Failed to copy '{} -> {}': {}"
    386 400 
                             .format(src, dest, e))
    ... ... @@ -743,11 +757,12 @@ def _ensure_real_directory(root, destpath):
    743 757 
     #    result: The FileListResult
    744 758 
     #    ignore_missing: Dont raise any error if a source file is missing
    745 759 
     #    presorted: Whether the passed list is known to be presorted
    760 
    +#    deterministic: Whether to use deterministic file attributes
    746 761 
     #
    747 762 
     #
    748 763 
     def _process_list(srcdir, destdir, filelist, actionfunc, result,
    749 764 
                       ignore_missing=False, report_written=False,
    750 
    -                  presorted=False):
    765 
    +                  presorted=False, deterministic=False, **kwargs):
    751 766
    752 767 
         # Keep track of directory permissions, since these need to be set
    753 768 
         # *after* files have been written.
    ... ... @@ -817,7 +832,8 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
    817 832 
                     result.ignored.append(path)
    818 833 
                     continue
    819 834
    820 
    -            actionfunc(srcpath, destpath, result=result)
    835 
    +            actionfunc(srcpath, destpath, result=result,
    836 
    +                       deterministic=deterministic, **kwargs)
    821 837
    822 838 
             elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
    823 839 
                 # Block or character device. Put contents of st_dev in a mknod.
    ... ... @@ -836,7 +852,10 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
    836 852
    837 853 
         # Write directory permissions now that all files have been written
    838 854 
         for d, perms in permissions:
    839 
    -        os.chmod(d, perms)
    855 
    +        if not deterministic:
    856 
    +            os.chmod(d, (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) & ~(stat.S_IWGRP | stat.S_IWOTH))
    857 
    +        else:
    858 
    +            os.chmod(d, perms)
    840 859
    841 860
    842 861 
     # _relative_symlink_target()
    ... ... @@ -998,18 +1017,29 @@ def _kill_process_tree(pid):
    998 1017 
     # Args:
    999 1018 
     #    popenargs (list): Popen() arguments
    1000 1019 
     #    terminate (bool): Whether to attempt graceful termination before killing
    1020 
    +#    deterministic_umask (bool): Whether to set umask to 0o022
    1001 1021 
     #    rest_of_args (kwargs): Remaining arguments to subprocess.call()
    1002 1022 
     #
    1003 1023 
     # Returns:
    1004 1024 
     #    (int): The process exit code.
    1005 1025 
     #    (str): The program output.
    1006 1026 
     #
    1007 
    -def _call(*popenargs, terminate=False, **kwargs):
    1027 
    +def _call(*popenargs, terminate=False, deterministic_umask=False, **kwargs):
    1008 1028
    1009 1029 
         kwargs['start_new_session'] = True
    1010 1030
    1011 1031 
         process = None
    1012 1032
    1033 
    +    if deterministic_umask:
    1034 
    +        old_preexec_fn = kwargs.get('preexec_fn')
    1035 
    +
    1036 
    +        def preexec_fn():
    1037 
    +            os.umask(stat.S_IWGRP | stat.S_IWOTH)
    1038 
    +            if old_preexec_fn is not None:
    1039 
    +                old_preexec_fn()
    1040 
    +
    1041 
    +        kwargs['preexec_fn'] = preexec_fn
    1042 
    +
    1013 1043 
         # Handle termination, suspend and resume
    1014 1044 
         def kill_proc():
    1015 1045 
             if process:

  • tests/integration/source-determinism.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream import _yaml, utils
    5 
    +from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    6 
    +
    7 
    +
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project"
    11 
    +)
    12 
    +
    13 
    +
    14 
    +def create_test_file(*path, mode=0o644, content='content\n'):
    15 
    +    path = os.path.join(*path)
    16 
    +    os.makedirs(os.path.dirname(path), exist_ok=True)
    17 
    +    with open(path, 'w') as f:
    18 
    +        f.write(content)
    19 
    +        os.fchmod(f.fileno(), mode)
    20 
    +
    21 
    +
    22 
    +def create_test_directory(*path, mode=0o644):
    23 
    +    create_test_file(*path, '.keep', content='')
    24 
    +    path = os.path.join(*path)
    25 
    +    os.chmod(path, mode)
    26 
    +
    27 
    +
    28 
    +@pytest.mark.integration
    29 
    +@pytest.mark.datafiles(DATA_DIR)
    30 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    31 
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    32 
    +    project = str(datafiles)
    33 
    +    element_name = 'list'
    34 
    +    element_path = os.path.join(project, 'elements', element_name)
    35 
    +    repodir = os.path.join(str(tmpdir), 'repo')
    36 
    +    sourcedir = os.path.join(project, 'source')
    37 
    +
    38 
    +    create_test_file(sourcedir, 'a.txt', mode=0o700)
    39 
    +    create_test_file(sourcedir, 'b.txt', mode=0o755)
    40 
    +    create_test_file(sourcedir, 'c.txt', mode=0o600)
    41 
    +    create_test_file(sourcedir, 'd.txt', mode=0o400)
    42 
    +    create_test_file(sourcedir, 'e.txt', mode=0o644)
    43 
    +    create_test_file(sourcedir, 'f.txt', mode=0o4755)
    44 
    +    create_test_file(sourcedir, 'g.txt', mode=0o2755)
    45 
    +    create_test_file(sourcedir, 'h.txt', mode=0o1755)
    46 
    +    create_test_directory(sourcedir, 'dir-a', mode=0o0700)
    47 
    +    create_test_directory(sourcedir, 'dir-c', mode=0o0755)
    48 
    +    create_test_directory(sourcedir, 'dir-d', mode=0o4755)
    49 
    +    create_test_directory(sourcedir, 'dir-e', mode=0o2755)
    50 
    +    create_test_directory(sourcedir, 'dir-f', mode=0o1755)
    51 
    +
    52 
    +    if kind == 'local':
    53 
    +        source = {'kind': 'local',
    54 
    +                  'path': 'source'}
    55 
    +    else:
    56 
    +        repo = create_repo(kind, repodir)
    57 
    +        ref = repo.create(sourcedir)
    58 
    +        source = repo.source_config(ref=ref)
    59 
    +    element = {
    60 
    +        'kind': 'manual',
    61 
    +        'depends': [
    62 
    +            {
    63 
    +                'filename': 'base.bst',
    64 
    +                'type': 'build'
    65 
    +            }
    66 
    +        ],
    67 
    +        'sources': [
    68 
    +            source
    69 
    +        ],
    70 
    +        'config': {
    71 
    +            'install-commands': [
    72 
    +                'ls -l >"%{install-root}/ls-l"'
    73 
    +            ]
    74 
    +        }
    75 
    +    }
    76 
    +    _yaml.dump(element, element_path)
    77 
    +
    78 
    +    def get_value_for_umask(umask):
    79 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
    80 
    +
    81 
    +        old_umask = os.umask(umask)
    82 
    +
    83 
    +        try:
    84 
    +            result = cli.run(project=project, args=['build', element_name])
    85 
    +            result.assert_success()
    86 
    +
    87 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    88 
    +            result.assert_success()
    89 
    +
    90 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    91 
    +                return f.read()
    92 
    +        finally:
    93 
    +            os.umask(old_umask)
    94 
    +            cli.remove_artifact_from_cache(project, element_name)
    95 
    +
    96 
    +    assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    97 
    +
    98 
    +
    99 
    +@pytest.mark.integration
    100 
    +@pytest.mark.datafiles(DATA_DIR)
    101 
    +def test_deterministic_source_local(cli, tmpdir, datafiles):
    102 
    +    """Only user rights should be considered for local source.
    103 
    +    """
    104 
    +    project = str(datafiles)
    105 
    +    element_name = 'test'
    106 
    +    element_path = os.path.join(project, 'elements', element_name)
    107 
    +    sourcedir = os.path.join(project, 'source')
    108 
    +
    109 
    +    element = {
    110 
    +        'kind': 'manual',
    111 
    +        'depends': [
    112 
    +            {
    113 
    +                'filename': 'base.bst',
    114 
    +                'type': 'build'
    115 
    +            }
    116 
    +        ],
    117 
    +        'sources': [
    118 
    +            {
    119 
    +                'kind': 'local',
    120 
    +                'path': 'source'
    121 
    +            }
    122 
    +        ],
    123 
    +        'config': {
    124 
    +            'install-commands': [
    125 
    +                'ls -l >"%{install-root}/ls-l"'
    126 
    +            ]
    127 
    +        }
    128 
    +    }
    129 
    +    _yaml.dump(element, element_path)
    130 
    +
    131 
    +    def get_value_for_mask(mask):
    132 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
    133 
    +
    134 
    +        create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
    135 
    +        create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
    136 
    +        create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
    137 
    +        create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
    138 
    +        create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
    139 
    +        create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
    140 
    +        create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
    141 
    +        create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
    142 
    +        create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
    143 
    +        try:
    144 
    +            result = cli.run(project=project, args=['build', element_name])
    145 
    +            result.assert_success()
    146 
    +
    147 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    148 
    +            result.assert_success()
    149 
    +
    150 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    151 
    +                return f.read()
    152 
    +        finally:
    153 
    +            cli.remove_artifact_from_cache(project, element_name)
    154 
    +
    155 
    +    assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]