[Notes] [Git][BuildStream/buildstream][willsalmon/CacheExpiryTest] 7 com

Will Salmon pushed to branch willsalmon/CacheExpiryTest at BuildStream / buildstream

Commits:

ed653fbc

by Chandan Singh at 2018-08-08T00:06:28Z

.gitlab-ci.yml: Remove unused script "install.sh"

65f382f1

by Chandan Singh at 2018-08-08T10:52:32Z

Merge branch 'chandan/remove-dead-code' into 'master'

.gitlab-ci.yml: Remove unused script "install.sh"

See merge request BuildStream/buildstream!617

c68dcab8

by Tiago Gomes at 2018-08-09T09:18:43Z

cascache: move tmp directory one level up

The CAS uses a temp directory while manipulating the cache, temporary
files can be added and removed while adding artifacts to the cache here.

Since calculation of the cache size happens in parallel to artifact
cache additions, this causes race conditions in the size calculation
job, as we end up calling `stat` on temporary files which are being
removed in parallel.

Handle this by moving the temporary directory out of the way, and avoid
considering the tmp directory when calculating the cache size

35ab0335

by Tiago Gomes at 2018-08-09T10:16:24Z

Merge branch 'tiagogomes/issue-520' into 'master'

Fix race condition when calculating disk usage

See merge request BuildStream/buildstream!600

8aa33e23

by Valentin David at 2018-08-09T12:49:17Z

Keep original flags for create in SafeHardlinks.

When open(2) is used with flags O_CREAT|O_RDWR, the file descriptor
must be readable. Unfortunately O_RDWR was not passed which made
read fail with EBADF and mmap to signal SIGBUS.

This issue happened with man-db for example.

Fixes #143.

ef7810f3

by Valentin David at 2018-08-09T14:06:50Z

Merge branch 'valentindavid/fuse-create-flags' into 'master'

Keep original flags for create in SafeHardlinks.

Closes #143

See merge request BuildStream/buildstream!624

9b29fafd

by William Salmon at 2018-08-09T16:31:19Z

Trying to mitigate a mtime granularity braking the cache tests

This patch mitigates the granularity of the mtimes used on the default gitlab
runners, Allowing the test sweat to pass on these runners.

11 changed files:

.gitlab-ci.yml
buildstream/_artifactcache/artifactcache.py
buildstream/_artifactcache/cascache.py
buildstream/_artifactcache/casserver.py
buildstream/_fuse/fuse.py
buildstream/_fuse/hardlinks.py
tests/artifactcache/expiry.py
tests/frontend/push.py
tests/frontend/workspace.py
tests/testutils/__init__.py
+ tests/testutils/runner_integration.py

Changes:

.gitlab-ci.yml

@@ -26,15 +26,6 @@ source_dist:
    - tar -ztf dist/*
    - tarball=$(cd dist && echo $(ls *))
 -  # Create an installer script
 -  - |
 -    cat > dist/install.sh << EOF
 -    #!/bin/sh
 -    tar -zxf ${tarball}
 -    cd ${tarball%.tar.gz}
 -    pip3 install --no-index .
 -    EOF
+-
    # unpack tarball as `dist/buildstream` directory
    - |
      cat > dist/unpack.sh << EOF
@@ -44,7 +35,6 @@ source_dist:
      EOF
    # Make our helpers executable
 -  - chmod +x dist/install.sh
    - chmod +x dist/unpack.sh
    artifacts:
      paths:

buildstream/_artifactcache/artifactcache.py

@@ -80,6 +80,8 @@ class ArtifactCache():
          self.context = context
          self.required_artifacts = set()
          self.extractdir = os.path.join(context.artifactdir, 'extract')
 +        self.tmpdir = os.path.join(context.artifactdir, 'tmp')
++
          self.max_size = context.cache_quota
          self.estimated_size = None
@@ -89,7 +91,8 @@ class ArtifactCache():
          self._local = False
          self.cache_size = None
 -        os.makedirs(context.artifactdir, exist_ok=True)
 +        os.makedirs(self.extractdir, exist_ok=True)
 +        os.makedirs(self.tmpdir, exist_ok=True)
      ################################################
      #  Methods implemented on the abstract class   #

buildstream/_artifactcache/cascache.py

@@ -56,7 +56,8 @@ class CASCache(ArtifactCache):
          super().__init__(context)
          self.casdir = os.path.join(context.artifactdir, 'cas')
 -        os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
 +        os.makedirs(os.path.join(self.casdir, 'refs', 'heads'), exist_ok=True)
 +        os.makedirs(os.path.join(self.casdir, 'objects'), exist_ok=True)
          self._enable_push = enable_push
@@ -85,8 +86,6 @@ class CASCache(ArtifactCache):
              # artifact has already been extracted
              return dest
 -        os.makedirs(self.extractdir, exist_ok=True)
+-
          with tempfile.TemporaryDirectory(prefix='tmp', dir=self.extractdir) as tmpdir:
              checkoutdir = os.path.join(tmpdir, ref)
              self._checkout(checkoutdir, tree)
@@ -394,7 +393,7 @@ class CASCache(ArtifactCache):
          try:
              h = hashlib.sha256()
              # Always write out new file to avoid corruption if input file is modified
 -            with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
 +            with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
                  # Set mode bits to 0644
                  os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
@@ -764,7 +763,7 @@ class CASCache(ArtifactCache):
              # already in local cache
              return
 -        with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
 +        with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
              self._fetch_blob(remote, tree, out)
              directory = remote_execution_pb2.Directory()
@@ -778,7 +777,7 @@ class CASCache(ArtifactCache):
                      # already in local cache
                      continue
 -                with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
 +                with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
                      self._fetch_blob(remote, filenode.digest, f)
                      digest = self.add_object(path=f.name)

buildstream/_artifactcache/casserver.py

@@ -161,7 +161,7 @@ class _ByteStreamServicer(bytestream_pb2_grpc.ByteStreamServicer):
          offset = 0
          finished = False
          resource_name = None
 -        with tempfile.NamedTemporaryFile(dir=os.path.join(self.cas.casdir, 'tmp')) as out:
 +        with tempfile.NamedTemporaryFile(dir=self.cas.tmpdir) as out:
              for request in request_iterator:
                  assert not finished
                  assert request.write_offset == offset

buildstream/_fuse/fuse.py

@@ -757,7 +757,11 @@ class FUSE(object):
          if self.raw_fi:
              return self.operations('create', path, mode, fi)
          else:
 -            fi.fh = self.operations('create', path, mode)
 +            # This line is different from upstream to fix issues
 +            # reading file opened with O_CREAT|O_RDWR.
 +            # See issue #143.
 +            fi.fh = self.operations('create', path, mode, fi.flags)
 +            # END OF MODIFICATION
              return 0
      def ftruncate(self, path, length, fip):

buildstream/_fuse/hardlinks.py

@@ -185,12 +185,12 @@ class SafeHardlinkOps(Operations):
          return os.open(full_path, flags)
 -    def create(self, path, mode, fi=None):
 +    def create(self, path, mode, flags):
          full_path = self._full_path(path)
          # If it already exists, ensure it's a copy first
          self._ensure_copy(full_path)
 -        return os.open(full_path, os.O_WRONLY | os.O_CREAT, mode)
 +        return os.open(full_path, flags, mode)
      def read(self, path, length, offset, fh):
          os.lseek(fh, offset, os.SEEK_SET)

tests/artifactcache/expiry.py

@@ -5,7 +5,7 @@ import pytest
  from buildstream import _yaml
  from buildstream._exceptions import ErrorDomain, LoadErrorReason
 -from tests.testutils import cli, create_element_size
 +from tests.testutils import cli, create_element_size, wait_for_cache_granularity
  DATA_DIR = os.path.join(
@@ -108,6 +108,8 @@ def test_expiry_order(cli, datafiles, tmpdir):
      res = cli.run(project=project, args=['build', 'target2.bst'])
      res.assert_success()
 +    wait_for_cache_granularity()
++
      # Now extract dep.bst
      res = cli.run(project=project, args=['checkout', 'dep.bst', checkout])
      res.assert_success()

tests/frontend/push.py

@@ -3,7 +3,7 @@ import pytest
  from buildstream._exceptions import ErrorDomain
  from tests.testutils import cli, create_artifact_share, create_element_size
 -from tests.testutils import generate_junction
 +from tests.testutils import generate_junction, wait_for_cache_granularity
  from . import configure_project
@@ -327,6 +327,8 @@ def test_recently_pulled_artifact_does_not_expire(cli, datafiles, tmpdir):
          # Ensure element1 is cached locally
          assert cli.get_element_state(project, 'element1.bst') == 'cached'
 +        wait_for_cache_granularity()
++
          # Create and build the element3 (of 5 MB)
          create_element_size('element3.bst', project, element_path, [], int(5e6))
          result = cli.run(project=project, args=['build', 'element3.bst'])

tests/frontend/workspace.py

@@ -3,7 +3,7 @@ import pytest
  import shutil
  import subprocess
  from ruamel.yaml.comments import CommentedSet
 -from tests.testutils import cli, create_repo, ALL_REPO_KINDS
 +from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity
  from buildstream import _yaml
  from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason
@@ -466,6 +466,8 @@ def test_detect_modifications(cli, tmpdir, datafiles, modification, strict):
      assert cli.get_element_state(project, element_name) == 'cached'
      assert cli.get_element_key(project, element_name) != "{:?<64}".format('')
 +    wait_for_cache_granularity()
++
      # Modify the workspace in various different ways, ensuring we
      # properly detect the changes.
+     #

tests/testutils/__init__.py

@@ -3,3 +3,4 @@ from .repo import create_repo, ALL_REPO_KINDS
  from .artifactshare import create_artifact_share
  from .element_generators import create_element_size
  from .junction import generate_junction
 +from .runner_integration import wait_for_cache_granularity

tests/testutils/runner_integration.py

 +import time
++
++
 +def wait_for_cache_granularity():
 +    # This isn't called very often so has minimal impact on test runtime.
 +    # If this changes it may be worth while adding a more sophisticated approach.
 +    """
 +    Mitigate the coarse granularity of the gitlab runners mtime
++
 +    This function waits for the mtime to increment so that the cache can sort by mtime and
 +    get the most recent results.
 +    """
 +    time.sleep(1.1)

[Notes] [Git][BuildStream/buildstream][willsalmon/CacheExpiryTest] 7 commits: .gitlab-ci.yml: Remove unused script "install.sh"

Will Salmon pushed to branch willsalmon/CacheExpiryTest at BuildStream / buildstream

Commits:

11 changed files:

Changes: