GNOME.org
 

[Notes] [Git][BuildStream/buildstream][valentindavid/deterministic-source] 2 commits: buildstream/plugins/sources/remote.py: Make staging deterministic.

Title: GitLab

Valentin David pushed to branch valentindavid/deterministic-source at BuildStream / buildstream

Commits:

2 changed files:

Changes:

  • buildstream/plugins/sources/remote.py
    ... ... @@ -74,7 +74,7 @@ class RemoteSource(DownloadableFileSource):
    74 74 
             # are not write protected in the sandbox.
    75 75 
             dest = os.path.join(directory, self.filename)
    76 76 
             with self.timed_activity("Staging remote file to {}".format(dest)):
    77 
    -            utils.safe_copy(self._get_mirror_file(), dest)
    77 
    +            utils.safe_copy(self._get_mirror_file(), dest, deterministic=True)
    78 78
    79 79
    80 80 
     def setup():

  • tests/integration/source-determinism.py
    1 
    +import os
    2 
    +import pytest
    3 
    +
    4 
    +from buildstream import _yaml, utils
    5 
    +from tests.testutils import cli, create_repo, ALL_REPO_KINDS
    6 
    +
    7 
    +
    8 
    +DATA_DIR = os.path.join(
    9 
    +    os.path.dirname(os.path.realpath(__file__)),
    10 
    +    "project"
    11 
    +)
    12 
    +
    13 
    +
    14 
    +def create_test_file(*path, mode=0o644, content='content\n'):
    15 
    +    path = os.path.join(*path)
    16 
    +    os.makedirs(os.path.dirname(path), exist_ok=True)
    17 
    +    with open(path, 'w') as f:
    18 
    +        f.write(content)
    19 
    +        os.fchmod(f.fileno(), mode)
    20 
    +
    21 
    +
    22 
    +def create_test_directory(*path, mode=0o644):
    23 
    +    create_test_file(*path, '.keep', content='')
    24 
    +    path = os.path.join(*path)
    25 
    +    os.chmod(path, mode)
    26 
    +
    27 
    +
    28 
    +@pytest.mark.integration
    29 
    +@pytest.mark.datafiles(DATA_DIR)
    30 
    +@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS] + ['local'])
    31 
    +def test_deterministic_source_umask(cli, tmpdir, datafiles, kind):
    32 
    +    project = str(datafiles)
    33 
    +    element_name = 'list'
    34 
    +    element_path = os.path.join(project, 'elements', element_name)
    35 
    +    repodir = os.path.join(str(tmpdir), 'repo')
    36 
    +    sourcedir = os.path.join(project, 'source')
    37 
    +
    38 
    +    create_test_file(sourcedir, 'a.txt', mode=0o700)
    39 
    +    create_test_file(sourcedir, 'b.txt', mode=0o755)
    40 
    +    create_test_file(sourcedir, 'c.txt', mode=0o600)
    41 
    +    create_test_file(sourcedir, 'd.txt', mode=0o400)
    42 
    +    create_test_file(sourcedir, 'e.txt', mode=0o644)
    43 
    +    create_test_file(sourcedir, 'f.txt', mode=0o4755)
    44 
    +    create_test_file(sourcedir, 'g.txt', mode=0o2755)
    45 
    +    create_test_file(sourcedir, 'h.txt', mode=0o1755)
    46 
    +    create_test_directory(sourcedir, 'dir-a', mode=0o0700)
    47 
    +    create_test_directory(sourcedir, 'dir-c', mode=0o0755)
    48 
    +    create_test_directory(sourcedir, 'dir-d', mode=0o4755)
    49 
    +    create_test_directory(sourcedir, 'dir-e', mode=0o2755)
    50 
    +    create_test_directory(sourcedir, 'dir-f', mode=0o1755)
    51 
    +
    52 
    +    if kind == 'local':
    53 
    +        source = {'kind': 'local',
    54 
    +                  'path': 'source'}
    55 
    +    else:
    56 
    +        repo = create_repo(kind, repodir)
    57 
    +        ref = repo.create(sourcedir)
    58 
    +        source = repo.source_config(ref=ref)
    59 
    +    element = {
    60 
    +        'kind': 'manual',
    61 
    +        'depends': [
    62 
    +            {
    63 
    +                'filename': 'base.bst',
    64 
    +                'type': 'build'
    65 
    +            }
    66 
    +        ],
    67 
    +        'sources': [
    68 
    +            source
    69 
    +        ],
    70 
    +        'config': {
    71 
    +            'install-commands': [
    72 
    +                'ls -l >"%{install-root}/ls-l"'
    73 
    +            ]
    74 
    +        }
    75 
    +    }
    76 
    +    _yaml.dump(element, element_path)
    77 
    +
    78 
    +    def get_value_for_umask(umask):
    79 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask))
    80 
    +
    81 
    +        old_umask = os.umask(umask)
    82 
    +
    83 
    +        try:
    84 
    +            result = cli.run(project=project, args=['build', element_name])
    85 
    +            result.assert_success()
    86 
    +
    87 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    88 
    +            result.assert_success()
    89 
    +
    90 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    91 
    +                return f.read()
    92 
    +        finally:
    93 
    +            os.umask(old_umask)
    94 
    +            cli.remove_artifact_from_cache(project, element_name)
    95 
    +
    96 
    +    assert get_value_for_umask(0o022) == get_value_for_umask(0o077)
    97 
    +
    98 
    +
    99 
    +@pytest.mark.integration
    100 
    +@pytest.mark.datafiles(DATA_DIR)
    101 
    +def test_deterministic_source_local(cli, tmpdir, datafiles):
    102 
    +    """Only user rights should be considered for local source.
    103 
    +    """
    104 
    +    project = str(datafiles)
    105 
    +    element_name = 'test'
    106 
    +    element_path = os.path.join(project, 'elements', element_name)
    107 
    +    sourcedir = os.path.join(project, 'source')
    108 
    +
    109 
    +    element = {
    110 
    +        'kind': 'manual',
    111 
    +        'depends': [
    112 
    +            {
    113 
    +                'filename': 'base.bst',
    114 
    +                'type': 'build'
    115 
    +            }
    116 
    +        ],
    117 
    +        'sources': [
    118 
    +            {
    119 
    +                'kind': 'local',
    120 
    +                'path': 'source'
    121 
    +            }
    122 
    +        ],
    123 
    +        'config': {
    124 
    +            'install-commands': [
    125 
    +                'ls -l >"%{install-root}/ls-l"'
    126 
    +            ]
    127 
    +        }
    128 
    +    }
    129 
    +    _yaml.dump(element, element_path)
    130 
    +
    131 
    +    def get_value_for_mask(mask):
    132 
    +        checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(mask))
    133 
    +
    134 
    +        create_test_file(sourcedir, 'a.txt', mode=0o644 & mask)
    135 
    +        create_test_file(sourcedir, 'b.txt', mode=0o755 & mask)
    136 
    +        create_test_file(sourcedir, 'c.txt', mode=0o4755 & mask)
    137 
    +        create_test_file(sourcedir, 'd.txt', mode=0o2755 & mask)
    138 
    +        create_test_file(sourcedir, 'e.txt', mode=0o1755 & mask)
    139 
    +        create_test_directory(sourcedir, 'dir-a', mode=0o0755 & mask)
    140 
    +        create_test_directory(sourcedir, 'dir-b', mode=0o4755 & mask)
    141 
    +        create_test_directory(sourcedir, 'dir-c', mode=0o2755 & mask)
    142 
    +        create_test_directory(sourcedir, 'dir-d', mode=0o1755 & mask)
    143 
    +        try:
    144 
    +            result = cli.run(project=project, args=['build', element_name])
    145 
    +            result.assert_success()
    146 
    +
    147 
    +            result = cli.run(project=project, args=['checkout', element_name, checkoutdir])
    148 
    +            result.assert_success()
    149 
    +
    150 
    +            with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f:
    151 
    +                return f.read()
    152 
    +        finally:
    153 
    +            cli.remove_artifact_from_cache(project, element_name)
    154 
    +
    155 
    +    assert get_value_for_mask(0o7777) == get_value_for_mask(0o0700)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]