[Notes] [Git][BuildStream/buildstream][jjardon/pycodestyle

Javier Jardón pushed to branch jjardon/pycodestyle_test at BuildStream / buildstream

Commits:

e0bb71b2

by Tiago Gomes at 2018-09-10T15:27:53Z

Merge branch 'tiagogomes/issue-287' into 'master'

Add validation of configuration variables

See merge request BuildStream/buildstream!678

47f3064a

by Jürg Billeter at 2018-09-10T15:43:25Z

_artifactcache/casserver.py: Implement Capabilities service

1a7fb3cb

by Jürg Billeter at 2018-09-10T15:43:25Z

_artifactcache/casserver.py: Implement BatchReadBlobs

Fixes #632.

b3ffcdc8

by Jürg Billeter at 2018-09-10T16:07:30Z

Merge branch 'juerg/cas-batch' into 'master'

_artifactcache/casserver.py: Implement BatchReadBlobs

Closes #632

See merge request BuildStream/buildstream!785

501060b3

by Javier Jardón at 2018-09-13T07:22:38Z

Use pycodestyle instead pep8

pep8 package was renamed to pycodestyle to reduce confusion.

a1bef8c4

by Javier Jardón at 2018-09-13T07:22:38Z

setup.cfg: Ignore W605 and W504 for now

This is the current occurence:
40      W605 invalid escape sequence '\s'
75      W504 line break after binary operator

96609e72
by Javier Jardón at 2018-09-13T07:25:24Z
```
WIP
```

Changes:

.gitlab-ci.yml

@@ -84,25 +84,25 @@ source_dist:
      - coverage-linux/
  tests-debian-9:
 -  image: buildstream/testsuite-debian:9-master-117-aa3a33b3
 +  image: buildstream/testsuite-debian:9-master-117-b3ffcdc8
    <<: *linux-tests
  tests-fedora-27:
 -  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
 +  image: buildstream/testsuite-fedora:27-master-117-b3ffcdc8
    <<: *linux-tests
  tests-fedora-28:
 -  image: buildstream/testsuite-fedora:28-master-117-aa3a33b3
 +  image: buildstream/testsuite-fedora:28-master-117-b3ffcdc8
    <<: *linux-tests
  tests-ubuntu-18.04:
 -  image: buildstream/testsuite-ubuntu:18.04-master-117-aa3a33b3
 +  image: buildstream/testsuite-ubuntu:18.04-master-117-b3ffcdc8
    <<: *linux-tests
  tests-unix:
    # Use fedora here, to a) run a test on fedora and b) ensure that we
    # can get rid of ostree - this is not possible with debian-8
 -  image: buildstream/testsuite-fedora:27-master-117-aa3a33b3
 +  image: buildstream/testsuite-fedora:27-master-117-b3ffcdc8
    stage: test
    variables:
      BST_FORCE_BACKEND: "unix"

buildstream/_artifactcache/casserver.py

@@ -38,6 +38,10 @@ from .._context import Context
  from .cascache import CASCache
 +# The default limit for gRPC messages is 4 MiB
 +_MAX_BATCH_TOTAL_SIZE_BYTES = 4 * 1024 * 1024
++
++
  # Trying to push an artifact that is too large
  class ArtifactTooLargeException(Exception):
      pass
@@ -67,6 +71,9 @@ def create_server(repo, *, enable_push):
      remote_execution_pb2_grpc.add_ContentAddressableStorageServicer_to_server(
          _ContentAddressableStorageServicer(artifactcache), server)
 +    remote_execution_pb2_grpc.add_CapabilitiesServicer_to_server(
 +        _CapabilitiesServicer(), server)
++
      buildstream_pb2_grpc.add_ReferenceStorageServicer_to_server(
          _ReferenceStorageServicer(artifactcache, enable_push=enable_push), server)
@@ -229,6 +236,48 @@ class _ContentAddressableStorageServicer(remote_execution_pb2_grpc.ContentAddres
                  d.size_bytes = digest.size_bytes
          return response
 +    def BatchReadBlobs(self, request, context):
 +        response = remote_execution_pb2.BatchReadBlobsResponse()
 +        batch_size = 0
++
 +        for digest in request.digests:
 +            batch_size += digest.size_bytes
 +            if batch_size > _MAX_BATCH_TOTAL_SIZE_BYTES:
 +                context.set_code(grpc.StatusCode.INVALID_ARGUMENT)
 +                return response
++
 +            blob_response = response.responses.add()
 +            blob_response.digest.hash = digest.hash
 +            blob_response.digest.size_bytes = digest.size_bytes
 +            try:
 +                with open(self.cas.objpath(digest), 'rb') as f:
 +                    if os.fstat(f.fileno()).st_size != digest.size_bytes:
 +                        blob_response.status.code = grpc.StatusCode.NOT_FOUND
 +                        continue
++
 +                    blob_response.data = f.read(digest.size_bytes)
 +            except FileNotFoundError:
 +                blob_response.status.code = grpc.StatusCode.NOT_FOUND
++
 +        return response
++
++
 +class _CapabilitiesServicer(remote_execution_pb2_grpc.CapabilitiesServicer):
 +    def GetCapabilities(self, request, context):
 +        response = remote_execution_pb2.ServerCapabilities()
++
 +        cache_capabilities = response.cache_capabilities
 +        cache_capabilities.digest_function.append(remote_execution_pb2.SHA256)
 +        cache_capabilities.action_cache_update_capabilities.update_enabled = False
 +        cache_capabilities.max_batch_total_size_bytes = _MAX_BATCH_TOTAL_SIZE_BYTES
 +        cache_capabilities.symlink_absolute_path_strategy = remote_execution_pb2.CacheCapabilities.ALLOWED
++
 +        response.deprecated_api_version.major = 2
 +        response.low_api_version.major = 2
 +        response.high_api_version.major = 2
++
 +        return response
++
  class _ReferenceStorageServicer(buildstream_pb2_grpc.ReferenceStorageServicer):
      def __init__(self, cas, *, enable_push):

buildstream/_ostree.py

@@ -34,7 +34,7 @@ from ._exceptions import BstError, ErrorDomain
  # pylint: disable=wrong-import-position,wrong-import-order
  gi.require_version('OSTree', '1.0')
 -from gi.repository import GLib, Gio, OSTree  # nopep8
 +from gi.repository import GLib, Gio, OSTree  # noqa
  # For users of this file, they must expect (except) it.

buildstream/element.py

@@ -1528,7 +1528,7 @@ class Element(Plugin):
                  utils._force_rmtree(rootdir)
              with _signals.terminator(cleanup_rootdir), \
 -                self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # nopep8
 +                self.__sandbox(rootdir, output_file, output_file, self.__sandbox_config) as sandbox:  # noqa
                  sandbox_vroot = sandbox.get_virtual_directory()

dev-requirements.txt

  coverage == 4.4.0
 -pep8
  pylint == 2.1.1
  pytest >= 3.7
 +pytest-codestyle >= 1.3.0
  pytest-cov >= 2.5.0
  pytest-datafiles
  pytest-env
 -pytest-pep8
  pytest-pylint
  pytest-xdist
  pytest-timeout

setup.cfg

@@ -11,20 +11,11 @@ parentdir_prefix = BuildStream-
  test=pytest
  [tool:pytest]
 -addopts = --verbose --basetemp ./tmp --pep8 --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
 +addopts = --verbose --basetemp ./tmp --codestyle --pylint --pylint-rcfile=.pylintrc --cov=buildstream --cov-config .coveragerc --durations=20
  norecursedirs = tests/integration/project integration-cache tmp __pycache__ .eggs
  python_files = tests/*/*.py
 -pep8maxlinelength = 119
 -pep8ignore =
 -    * E129
 -    * E125
 -    doc/source/conf.py ALL
 -    tmp/* ALL
 -    */lib/python3* ALL
 -    */bin/* ALL
 -    buildstream/_fuse/fuse.py ALL
 -    .eggs/* ALL
 -    *_pb2.py ALL
 -    *_pb2_grpc.py ALL
 +codestyle_max_line_length = 119
 +codestyle_ignore = E129 E125 W504 W605
 +codestyle_exclude = doc/source/conf.py buildstream/_fuse/fuse.py buildstream/_protos/build/bazel/remote/execution/v2/remote_execution_pb2.py buildstream/_protos/google/api/http_pb2.py buildstream/_protos/google/bytestream/bytestream_pb2.py buildstream/_protos/google/longrunning/operations_pb2.py buildstream/_protos/google/rpc/status_pb2.py
  env =
      D:BST_TEST_SUITE=True

[Notes] [Git][BuildStream/buildstream][jjardon/pycodestyle_test] 7 commits: Merge branch 'tiagogomes/issue-287' into 'master'

Javier Jardón pushed to branch jjardon/pycodestyle_test at BuildStream / buildstream

Commits:

6 changed files:

Changes: