[Notes] [Git][BuildStream/buildstream][tiagogomes/system

Tiago Gomes pushed to branch tiagogomes/system_tests at BuildStream / buildstream

Commits:

4b1c6447

by Tiago Gomes at 2018-08-07T12:37:26Z

artifactcache: turn method safe concurrently

Change ArtifactCache.get_approximate_cache_size() to be safe to use
concurrently without the caller having to have exclusive access to the
cache resource.

d98a2e20

by Tiago Gomes at 2018-08-07T12:37:29Z

cascache: move tmp directory one level up

The tmp directory is filled when adding an artifact with temporary
files. This causes problems on calculate_cache_size() because we could
try to a attemp to do a stat() on a temporary file that meanwhile has
been reaped.

Handle this by not considering the tmp directory when calculating the
cache size.

7dd49729
by Tiago Gomes at 2018-08-07T12:37:29Z
```
TMP: Ignore socket files on _process_list
```

44f9067f

by Tiago Gomes at 2018-08-07T12:37:29Z

WIP Add end-to-end tests

TODO:
* Uncomment when clausule.
* Remove debug information.

5 changed files:

.gitlab-ci.yml
buildstream/_artifactcache/artifactcache.py
buildstream/_artifactcache/cascache.py
buildstream/_scheduler/queues/buildqueue.py
buildstream/utils.py

Changes:

.gitlab-ci.yml

@@ -84,6 +84,8 @@ source_dist:
    - cd ../..
    - mkdir -p coverage-linux/
    - cp dist/buildstream/.coverage.* coverage-linux/coverage."${CI_JOB_NAME}"
 +  except:
 +  - schedules
    artifacts:
      paths:
      - coverage-linux/
@@ -125,6 +127,8 @@ tests-unix:
      - cd ../..
      - mkdir -p coverage-unix/
      - cp dist/buildstream/.coverage.* coverage-unix/coverage.unix
 +  except:
 +  - schedules
    artifacts:
      paths:
      - coverage-unix/
@@ -146,10 +150,53 @@ docs:
    - make BST_FORCE_SESSION_REBUILD=1 -C doc
    - cd ../..
    - mv dist/buildstream/doc/build/html public
 +  except:
 +  - schedules
    artifacts:
      paths:
      - public/
 +.system-tests: &system-tests-template
 +  stage: test
 +  variables:
 +    XDG_CACHE_HOME: /cache
 +    bst_ext_url: git+https://gitlab.com/BuildStream/bst-external.git
 +    bst_ext_ref: 31376f750df713b3613d5348f93100c7e0ef68c0 # 0.3
 +    fd_sdk_ref: 27cb7575f57e2922331facbd252dc690807c7182 # 18.08
 +    expected_cache_key: ceee9b5a
 +    expected_md5sum: 083966a5dc914d52e5245af3a06e933e
 +  before_script:
 +  - (cd dist && ./unpack.sh && cd buildstream && pip3 install .)
 +  - pip3 install --user -e ${bst_ext_url}@${bst_ext_ref}#egg=bst_ext
 +  - git clone https://gitlab.com/freedesktop-sdk/freedesktop-sdk.git
 +  - git -C freedesktop-sdk checkout ${fd_sdk_ref}
 +  #when:
 +  #- schedules
++
 +system-tests:
 +  <<: *system-tests-template
 +  script:
 +  - make -C freedesktop-sdk
 +  - pwd
 +  - cache_key=$(bst -C sdk show all.bst --deps none --format %{key})
 +  - test ${cache_key} == ${expected_cache_key}
 +  - bst -C sdk checkout --tar all.bst all.tar
 +  - checksum=$(md5sum all.tar)
 +  - test ${checksum} == ${expected_checksum}
++
 +system-tests-no-cache:
 +  <<: *system-tests-template
 +  script:
 +  - sed -i '/artifacts:/,+1 d' freedesktop-sdk/bootstrap/project.conf
 +  - sed -i '/artifacts:/,+1 d' freedesktop-sdk/project.conf
 +  - make -C freedesktop-sdk
 +  - pwd
 +  - cache_key=$(bst -C sdk show all.bst --deps none --format %{key})
 +  - test ${cache_key} == ${expected_cache_key}
 +  - bst -C sdk checkout --tar all.bst all.tar
 +  - checksum=$(md5sum all.tar)
 +  - test ${checksum} == ${expected_checksum}
++
  #####################################################
  #                    Post stage                     #
@@ -174,6 +221,8 @@ codequality:
          --volume "$PWD":/code
          --volume /var/run/docker.sock:/var/run/docker.sock
          "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
 +  except:
 +  - schedules
    artifacts:
      paths: [codeclimate.json]
@@ -199,6 +248,8 @@ analysis:
      radon raw -s -j buildstream > analysis/raw.json
      radon raw -s buildstream
 +  except:
 +  - schedules
    artifacts:
      paths:
      - analysis/
@@ -223,6 +274,8 @@ coverage:
    - tests-fedora-27
    - tests-unix
    - source_dist
 +  except:
 +  - schedules
  # Deploy, only for merges which land on master branch.
+ #
@@ -247,3 +300,5 @@ pages:
    # See https://gitlab.com/gitlab-org/gitlab-ce/issues/35141
+   #
    - master
 +  except:
 +  - schedules

buildstream/_artifactcache/artifactcache.py

@@ -276,11 +276,6 @@ class ArtifactCache():
      #     (int) An approximation of the artifact cache size.
+     #
      def get_approximate_cache_size(self):
 -        # If we don't currently have an estimate, figure out the real
 -        # cache size.
 -        if self.estimated_size is None:
 -            self.estimated_size = self.calculate_cache_size()
+-
          return self.estimated_size
      ################################################
@@ -534,10 +529,8 @@ class ArtifactCache():
      # ArtifactCache.estimated_size.
+     #
      def _add_artifact_size(self, artifact_size):
 -        if not self.estimated_size:
 -            self.estimated_size = self.calculate_cache_size()
+-
 -        self.estimated_size += artifact_size
 +        if self.estimated_size:
 +            self.estimated_size += artifact_size
      # _set_cache_size()
+     #

buildstream/_artifactcache/cascache.py

@@ -56,7 +56,9 @@ class CASCache(ArtifactCache):
          super().__init__(context)
          self.casdir = os.path.join(context.artifactdir, 'cas')
 -        os.makedirs(os.path.join(self.casdir, 'tmp'), exist_ok=True)
++
 +        self.tmpdir = os.path.join(context.artifactdir, 'tmp')
 +        os.makedirs(self.tmpdir, exist_ok=True)
          self._enable_push = enable_push
@@ -394,7 +396,7 @@ class CASCache(ArtifactCache):
          try:
              h = hashlib.sha256()
              # Always write out new file to avoid corruption if input file is modified
 -            with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
 +            with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
                  # Set mode bits to 0644
                  os.chmod(out.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
@@ -764,7 +766,7 @@ class CASCache(ArtifactCache):
              # already in local cache
              return
 -        with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as out:
 +        with tempfile.NamedTemporaryFile(dir=self.tmpdir) as out:
              self._fetch_blob(remote, tree, out)
              directory = remote_execution_pb2.Directory()
@@ -778,7 +780,7 @@ class CASCache(ArtifactCache):
                      # already in local cache
                      continue
 -                with tempfile.NamedTemporaryFile(dir=os.path.join(self.casdir, 'tmp')) as f:
 +                with tempfile.NamedTemporaryFile(dir=self.tmpdir) as f:
                      self._fetch_blob(remote, filenode.digest, f)
                      digest = self.add_object(path=f.name)

buildstream/_scheduler/queues/buildqueue.py

+ #
 -#  Copyright (C) 2016 Codethink Limited
 +#  Copyright (C) 2018 Codethink Limited
+ #
  #  This program is free software; you can redistribute it and/or
  #  modify it under the terms of the GNU Lesser General Public
@@ -56,13 +56,16 @@ class BuildQueue(Queue):
              return
          artifact_size = job.child_data.get('artifact_size', False)
 +        if not artifact_size:
 +            return
 -        if artifact_size:
 -            cache = element._get_artifact_cache()
 -            cache._add_artifact_size(artifact_size)
 +        cache = element._get_artifact_cache()
 +        cache._add_artifact_size(artifact_size)
 -            if cache.get_approximate_cache_size() > self._scheduler.context.cache_quota:
 -                self._scheduler._check_cache_size_real()
 +        approx_size = cache.get_approximate_cache_size()
 +        if not approx_size or (approx_size >
 +                    self._scheduler.context.cache_quota):
 +            self._scheduler._check_cache_size_real()
      def done(self, job, element, result, success):

buildstream/utils.py

@@ -827,6 +827,12 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
              os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
              os.chmod(destpath, file_stat.st_mode)
 +        elif stat.S_ISFIFO(mode):
 +            os.mkfifo(destpath, mode)
++
 +        elif stat.S_ISSOCK(mode):
 +            pass
++
          else:
              # Unsupported type.
              raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath))

[Notes] [Git][BuildStream/buildstream][tiagogomes/system_tests] 4 commits: artifactcache: turn method safe concurrently

Tiago Gomes pushed to branch tiagogomes/system_tests at BuildStream / buildstream

Commits:

5 changed files:

Changes: