GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/unittests-bot-interface] 3 commits: Fix CAS issues on Python 3.5

Title: GitLab

finnball pushed to branch finn/unittests-bot-interface at BuildGrid / buildgrid

Commits:

6 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -11,12 +11,11 @@ before_script:
    11 11 
       - export PATH=~/.local/bin:${PATH}
    12 12 
       - pip3 install --user -e .
    13 13
    14 
    -tests-fedora:
    14 
    +.tests-template: &linux-tests
    15 15 
       stage: test
    16 16 
       variables:
    17 17 
         PYTEST_ADDOPTS: "--color=yes"
    18 18 
       script:
    19 
    -    - yum -y install clang libffi-devel openssl-devel python3-devel
    20 19 
         - python3 setup.py test
    21 20 
         - mkdir -p coverage/
    22 21 
         - cp .coverage.* coverage/coverage."${CI_JOB_NAME}"
    ... ... @@ -24,15 +23,34 @@ tests-fedora:
    24 23 
         paths:
    25 24 
         - coverage/
    26 25
    27 
    -
    28 
    -tests-dummy-job-fedora:
    26 
    +.run-dummy-job-template: &dummy-job
    29 27 
       stage: test
    30 28 
       script:
    31 29 
         - ${BGD} server start &
    32 30 
         - sleep 1 # Allow server to boot
    33 
    -    - ${BGD} bot --host=0.0.0.0 dummy &
    31 
    +    - ${BGD} bot --host=0.0.0.0 --continuous dummy &
    34 32 
         - ${BGD} execute --host=0.0.0.0 request --wait-for-completion
    35 33
    34 
    +tests-debian:
    35 
    +  image: buildstream/buildstream-debian
    36 
    +  <<: *linux-tests
    37 
    +
    38 
    +# Need to yum install until we have our own image
    39 
    +tests-fedora:
    40 
    +  <<: *linux-tests
    41 
    +  script:
    42 
    +    - yum -y install clang libffi-devel openssl-devel python3-devel
    43 
    +    - python3 setup.py test
    44 
    +    - mkdir -p coverage/
    45 
    +    - cp .coverage.* coverage/coverage."${CI_JOB_NAME}"
    46 
    +
    47 
    +run-dummy-job-debian:
    48 
    +  image: buildstream/buildstream-debian
    49 
    +  <<: *dummy-job
    50 
    +
    51 
    +run-dummy-job-fedora:
    52 
    +  <<: *dummy-job
    53 
    +
    36 54 
     coverage:
    37 55 
       stage: post
    38 56 
       coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'

  • app/commands/cmd_bot.py
    ... ... @@ -43,15 +43,17 @@ from google.devtools.remoteexecution.v1test import remote_execution_pb2, remote_
    43 43 
     from google.protobuf import any_pb2
    44 44
    45 45 
     @click.group(short_help = 'Create a bot client')
    46 
    +@click.option('--continuous', is_flag=True)
    46 47 
     @click.option('--parent', default='bgd_test')
    47 48 
     @click.option('--number-of-leases', default=1)
    48 49 
     @click.option('--port', default='50051')
    49 50 
     @click.option('--host', default='localhost')
    50 51 
     @pass_context
    51 
    -def cli(context, host, port, number_of_leases, parent):
    52 
    +def cli(context, host, port, number_of_leases, parent, continuous):
    52 53 
         context.logger = logging.getLogger(__name__)
    53 54 
         context.logger.info("Starting on port {}".format(port))
    54 55
    56 
    +    context.continuous = continuous
    55 57 
         context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    56 58 
         context.number_of_leases = number_of_leases
    57 59 
         context.parent = parent
    ... ... @@ -71,7 +73,8 @@ def dummy(context):
    71 73 
                     context=context,
    72 74 
                     channel=context.channel,
    73 75 
                     parent=context.parent,
    74 
    -                number_of_leases=context.number_of_leases)
    76 
    +                number_of_leases=context.number_of_leases,
    77 
    +                continuous=context.continuous)
    75 78
    76 79 
         except KeyboardInterrupt:
    77 80 
             pass
    ... ... @@ -105,7 +108,8 @@ def _work_buildbox(context, remote, port, server_cert, client_key, client_cert,
    105 108 
                     context=context,
    106 109 
                     channel=context.channel,
    107 110 
                     parent=context.parent,
    108 
    -                number_of_leases=context.number_of_leases)
    111 
    +                number_of_leases=context.number_of_leases,
    112 
    +                continuous=context.continuous)
    109 113
    110 114 
         except KeyboardInterrupt:
    111 115 
             pass

  • buildgrid/bot/bot.py
    ... ... @@ -39,33 +39,31 @@ class Bot(object):
    39 39 
         Creates a local BotSession.
    40 40 
         """
    41 41
    42 
    -    def __init__(self, work, context, channel, parent, number_of_leases):
    42 
    +    def __init__(self, work, context, channel, parent, number_of_leases, continuous=True):
    43 43 
             if not inspect.iscoroutinefunction(work):
    44 44 
                 raise BotError("work function must be async")
    45 45
    46 
    +        print(type(context))
    47 
    +
    46 48 
             self.interface = bot_interface.BotInterface(channel)
    47 49 
             self.logger = logging.getLogger(__name__)
    48 50
    49 51 
             self._create_session(parent, number_of_leases)
    50 52 
             self._work_queue = queue.Queue(maxsize = number_of_leases)
    51 53
    52 
    -        try:
    53 
    -            while True:
    54 
    -                ## TODO: Leases independently finish
    55 
    -                ## Allow leases to queue finished work independently instead
    56 
    -                ## of waiting for all to finish
    57 
    -                futures = [self._do_work(work, context, lease) for lease in self._get_work()]
    58 
    -                if futures:
    59 
    -                    loop = asyncio.new_event_loop()
    60 
    -                    leases_complete, _ = loop.run_until_complete(asyncio.wait(futures))
    61 
    -                    work_complete = [(lease.result().assignment, lease.result(),) for lease in leases_complete]
    62 
    -                    self._work_complete(work_complete)
    63 
    -                    loop.close()
    64 
    -                self._update_bot_session()
    65 
    -                time.sleep(2)
    66 
    -        except Exception as e:
    67 
    -            self.logger.error(e)
    68 
    -            raise BotError(e)
    54 
    +        while continuous:
    55 
    +            ## TODO: Leases independently finish
    56 
    +            ## Allow leases to queue finished work independently instead
    57 
    +            ## of waiting for all to finish
    58 
    +            futures = [self._do_work(work, context, lease) for lease in self._get_work()]
    59 
    +            if futures:
    60 
    +                loop = asyncio.new_event_loop()
    61 
    +                leases_complete, _ = loop.run_until_complete(asyncio.wait(futures))
    62 
    +                work_complete = [(lease.result().assignment, lease.result(),) for lease in leases_complete]
    63 
    +                self._work_complete(work_complete)
    64 
    +                loop.close()
    65 
    +            self._update_bot_session()
    66 
    +            time.sleep(2)
    69 67
    70 68 
         @property
    71 69 
         def bot_session(self):
    ... ... @@ -97,7 +95,6 @@ class Bot(object):
    97 95 
         def _get_work(self):
    98 96 
             while not self._work_queue.empty():
    99 97 
                 yield self._work_queue.get()
    100 
    -            self._work_queue.task_done()
    101 98
    102 99 
         def _work_complete(self, leases_complete):
    103 100 
             """ Bot updates itself with any completed work.

  • buildgrid/server/cas/storage/disk.py
    ... ... @@ -32,14 +32,14 @@ class DiskStorage(StorageABC):
    32 32
    33 33 
         def __init__(self, path):
    34 34 
             self._path = pathlib.Path(path)
    35 
    -        os.makedirs(self._path / "temp", exist_ok=True)
    35 
    +        os.makedirs(str(self._path / "temp"), exist_ok=True)
    36 36
    37 37 
         def has_blob(self, digest):
    38 38 
             return (self._path / (digest.hash + "_" + str(digest.size_bytes))).exists()
    39 39
    40 40 
         def get_blob(self, digest):
    41 41 
             try:
    42 
    -            return open(self._path / (digest.hash + "_" + str(digest.size_bytes)), 'rb')
    42 
    +            return (self._path / (digest.hash + "_" + str(digest.size_bytes))).open('rb')
    43 43 
             except FileNotFoundError:
    44 44 
                 return None
    45 45
    ... ... @@ -49,7 +49,7 @@ class DiskStorage(StorageABC):
    49 49 
         def commit_write(self, digest, write_session):
    50 50 
             # Atomically move the temporary file into place.
    51 51 
             path = self._path / (digest.hash + "_" + str(digest.size_bytes))
    52 
    -        os.replace(write_session.name, path)
    52 
    +        os.replace(write_session.name, str(path))
    53 53 
             try:
    54 54 
                 write_session.close()
    55 55 
             except FileNotFoundError:

  • tests/cas/test_services.py
    ... ... @@ -83,7 +83,7 @@ def test_bytestream_read(data_to_read, instance):
    83 83 
         request = bytestream_pb2.ReadRequest()
    84 84 
         if instance != "":
    85 85 
             request.resource_name = instance + "/"
    86 
    -    request.resource_name += f"blobs/{HASH(data_to_read).hexdigest()}/{len(data_to_read)}"
    86 
    +    request.resource_name += "blobs/{}/{}".format(HASH(data_to_read).hexdigest(), len(data_to_read))
    87 87
    88 88 
         data = b""
    89 89 
         for response in servicer.Read(request, None):
    ... ... @@ -101,7 +101,7 @@ def test_bytestream_read_many(instance):
    101 101 
         request = bytestream_pb2.ReadRequest()
    102 102 
         if instance != "":
    103 103 
             request.resource_name = instance + "/"
    104 
    -    request.resource_name += f"blobs/{HASH(data_to_read).hexdigest()}/{len(data_to_read)}"
    104 
    +    request.resource_name += "blobs/{}/{}".format(HASH(data_to_read).hexdigest(), len(data_to_read))
    105 105
    106 106 
         data = b""
    107 107 
         for response in servicer.Read(request, None):
    ... ... @@ -119,7 +119,7 @@ def test_bytestream_write(instance, extra_data):
    119 119 
         if instance != "":
    120 120 
             resource_name = instance + "/"
    121 121 
         hash_ = HASH(b'abcdef').hexdigest()
    122 
    -    resource_name += f"uploads/UUID-HERE/blobs/{hash_}/6"
    122 
    +    resource_name += "uploads/UUID-HERE/blobs/{}/6".format(hash_)
    123 123 
         resource_name += extra_data
    124 124 
         requests = [
    125 125 
             bytestream_pb2.WriteRequest(resource_name=resource_name, data=b'abc'),
    ... ... @@ -139,7 +139,7 @@ def test_bytestream_write_rejects_wrong_hash():
    139 139
    140 140 
         data = b'some data'
    141 141 
         wrong_hash = HASH(b'incorrect').hexdigest()
    142 
    -    resource_name = f"uploads/UUID-HERE/blobs/{wrong_hash}/9"
    142 
    +    resource_name = "uploads/UUID-HERE/blobs/{}/9".format(wrong_hash)
    143 143 
         requests = [
    144 144 
             bytestream_pb2.WriteRequest(resource_name=resource_name, data=data, finish_write=True)
    145 145 
         ]

  • tests/integration/bot_interface.py
    1 
    +# Copyright (C) 2018 Codethink Limited
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +#
    15 
    +# Authors:
    16 
    +#        Finn Ball <finn ball codethink co uk>
    17 
    +
    18 
    +import copy
    19 
    +import grpc
    20 
    +import logging
    21 
    +import mock
    22 
    +import pytest
    23 
    +import uuid
    24 
    +
    25 
    +from buildgrid.bot import bot, bot_interface
    26 
    +
    27 
    +async def _work_dummy(context, lease):
    28 
    +    return lease
    29 
    +
    30 
    +class ContextMock():
    31 
    +    def __init__(self):
    32 
    +        self.logger = logging.getLogger(__name__)
    33 
    +
    34 
    +# GRPC context
    35 
    +@pytest.fixture
    36 
    +def context():
    37 
    +    yield ContextMock()
    38 
    +
    39 
    +# GRPC context
    40 
    +@pytest.fixture
    41 
    +def channel():
    42 
    +    yield mock.MagicMock(spec = grpc.insecure_channel(''))
    43 
    +
    44 
    +@pytest.fixture
    45 
    +def instance(channel):
    46 
    +    yield bot.Bot(work=_work_dummy,
    47 
    +                  context=ContextMock(),
    48 
    +                  channel=channel,
    49 
    +                  parent='rach',
    50 
    +                  number_of_leases=1,
    51 
    +                  continuous=False)
    52 
    +
    53 
    +def test_create_job(instance):
    54 
    +    instance.bot_session()

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]