[Notes] [Git][BuildStream/buildstream][jjardon/aarch64] 5 commits: eleme

Javier Jardón pushed to branch jjardon/aarch64 at BuildStream / buildstream

Commits:

261e2cd3

by Jim MacArthur at 2018-11-07T12:02:43Z

element.py: Use _force_rmtree instead of custom code.

There are arguments for and against using onerror - see issue #735
for details.

8931e42c

by Jim MacArthur at 2018-11-07T12:26:06Z

Merge branch 'jmac/make_writable_combination' into 'master'

Use _force_rmtree instead of custom code.

Closes #735

See merge request BuildStream/buildstream!927

6ccfab0b

by Valentin David at 2018-11-07T16:28:47Z

Fix cache corruption by scripts when layout and integration commands are used

Root directory was marked as a non-artifact mount, so not using
SafeHardLink.  However integration commands executed with write access
to the root directory.

Fixes #749

f24e20e9

by richardmaw-codethink at 2018-11-07T16:59:50Z

Merge branch 'valentindavid/script-artifact-corruption' into 'master'

Fix cache corruption by scripts when layout and integration commands are used

Closes #749

See merge request BuildStream/buildstream!934

0722ded2

by Javier Jardón at 2018-11-07T21:15:18Z

.gitlab-ci.yml: Add jobs to run tests in aarch64

8 changed files:

.gitlab-ci.yml
buildstream/element.py
buildstream/scriptelement.py
+ tests/integration/project/elements/script/corruption-image.bst
+ tests/integration/project/elements/script/corruption-integration.bst
+ tests/integration/project/elements/script/corruption.bst
+ tests/integration/project/files/canary
tests/integration/script.py

Changes:

.gitlab-ci.yml

@@ -89,18 +89,36 @@ tests-debian-9:
    image: buildstream/testsuite-debian:9-master-119-552f5fc6
    <<: *linux-tests
 +tests-debian-9-aarch64:
 +  image: buildstream/testsuite-debian:aarch64-9-master-123-8931e42c
 +  tags:
 +  - aarch64
 +  <<: *linux-tests
++
  tests-fedora-27:
 -  image: buildstream/testsuite-fedora:27-master-119-552f5fc6
 +  image: buildstream/testsuite-fedora:aarch64-27-master-123-8931e42c
    <<: *linux-tests
  tests-fedora-28:
    image: buildstream/testsuite-fedora:28-master-119-552f5fc6
    <<: *linux-tests
 +tests-fedora-28-aarch64:
 +  image: buildstream/testsuite-fedora:aarch64-27-master-123-8931e42c
 +  tags:
 +  - aarch64
 +  <<: *linux-tests
++
  tests-ubuntu-18.04:
    image: buildstream/testsuite-ubuntu:18.04-master-119-552f5fc6
    <<: *linux-tests
 +tests-ubuntu-18.04--aarch64:
 +  image: buildstream/testsuite-ubuntu:
 +  tags:
 +  - aarch64
 +  <<: *linux-tests
++
  tests-unix:
    # Use fedora here, to a) run a test on fedora and b) ensure that we
    # can get rid of ostree - this is not possible with debian-8

buildstream/element.py

@@ -1410,16 +1410,9 @@ class Element(Plugin):
              finally:
                  # Staging may produce directories with less than 'rwx' permissions
 -                # for the owner, which will break tempfile, so we need to use chmod
 -                # occasionally.
 -                def make_dir_writable(fn, path, excinfo):
 -                    os.chmod(os.path.dirname(path), 0o777)
 -                    if os.path.isdir(path):
 -                        os.rmdir(path)
 -                    else:
 -                        os.remove(path)
 -                shutil.rmtree(temp_staging_directory, onerror=make_dir_writable)
+-
 +                # for the owner, which breaks tempfile. _force_rmtree will deal
 +                # with these.
 +                utils._force_rmtree(temp_staging_directory)
          # Ensure deterministic mtime of sources at build time
          vdirectory.set_deterministic_mtime()
          # Ensure deterministic owners of sources at build time

buildstream/scriptelement.py

@@ -201,16 +201,17 @@ class ScriptElement(Element):
          # Setup environment
          sandbox.set_environment(self.get_environment())
 +        # Tell the sandbox to mount the install root
 +        directories = {'/': False}
++
          # Mark the artifact directories in the layout
          for item in self.__layout:
 -            if item['destination'] != '/':
 -                if item['element']:
 -                    sandbox.mark_directory(item['destination'], artifact=True)
 -                else:
 -                    sandbox.mark_directory(item['destination'])
 +            destination = item['destination']
 +            was_artifact = directories.get(destination, False)
 +            directories[destination] = item['element'] or was_artifact
 -        # Tell the sandbox to mount the install root
 -        sandbox.mark_directory(self.__install_root)
 +        for directory, artifact in directories.items():
 +            sandbox.mark_directory(directory, artifact=artifact)
      def stage(self, sandbox):

tests/integration/project/elements/script/corruption-image.bst

 +kind: import
 +sources:
 +- kind: local
 +  path: files/canary

tests/integration/project/elements/script/corruption-integration.bst

 +kind: stack
++
 +public:
 +  bst:
 +    integration-commands:
 +      - echo smashed >>/canary
++

tests/integration/project/elements/script/corruption.bst

 +kind: script
++
 +depends:
 +- filename: base.bst
 +  type: build
 +- filename: script/corruption-image.bst
 +  type: build
 +- filename: script/corruption-integration.bst
 +  type: build
++
 +variables:
 +  install-root: "/"
++
 +config:
 +  layout:
 +  - element: base.bst
 +    destination: "/"
 +  - element: script/corruption-image.bst
 +    destination: "/"
 +  - element: script/corruption-integration.bst
 +    destination: "/"

tests/integration/project/files/canary

+alive

tests/integration/script.py

@@ -155,3 +155,32 @@ def test_script_layout(cli, tmpdir, datafiles):
          text = f.read()
      assert text == "Hi\n"
++
++
 +@pytest.mark.datafiles(DATA_DIR)
 +def test_regression_cache_corruption(cli, tmpdir, datafiles):
 +    project = str(datafiles)
 +    checkout_original = os.path.join(cli.directory, 'checkout-original')
 +    checkout_after = os.path.join(cli.directory, 'checkout-after')
 +    element_name = 'script/corruption.bst'
 +    canary_element_name = 'script/corruption-image.bst'
++
 +    res = cli.run(project=project, args=['build', canary_element_name])
 +    assert res.exit_code == 0
++
 +    res = cli.run(project=project, args=['checkout', canary_element_name,
 +                                         checkout_original])
 +    assert res.exit_code == 0
++
 +    with open(os.path.join(checkout_original, 'canary')) as f:
 +        assert f.read() == 'alive\n'
++
 +    res = cli.run(project=project, args=['build', element_name])
 +    assert res.exit_code == 0
++
 +    res = cli.run(project=project, args=['checkout', canary_element_name,
 +                                         checkout_after])
 +    assert res.exit_code == 0
++
 +    with open(os.path.join(checkout_after, 'canary')) as f:
 +        assert f.read() == 'alive\n'

[Notes] [Git][BuildStream/buildstream][jjardon/aarch64] 5 commits: element.py: Use _force_rmtree instead of custom code.

Javier Jardón pushed to branch jjardon/aarch64 at BuildStream / buildstream

Commits:

8 changed files:

Changes: