GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/77-cas-uploader] tests/cas/test_client.py: Add unit-tests for CAS client

Title: GitLab

Martin Blanchard pushed to branch mablanch/77-cas-uploader at BuildGrid / buildgrid

Commits:

5 changed files:

Changes:

  • tests/cas/data/hello.cc
    1 
    +#include <iostream>
    2 
    +
    3 
    +int main()
    4 
    +{
    5 
    +  std::cout << "Hello, World!" << std::endl;
    6 
    +  return 0;
    7 
    +}

  • tests/cas/data/hello/hello.c
    1 
    +#include <stdio.h>
    2 
    +
    3 
    +#include "hello.h"
    4 
    +
    5 
    +int main()
    6 
    +{
    7 
    +  printf("%s\n", HELLO_WORLD);
    8 
    +  return 0;
    9 
    +}

  • tests/cas/data/hello/hello.h
    1 
    +#define HELLO_WORLD "Hello, World!"

  • tests/cas/data/void

  • tests/cas/test_client.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +# pylint: disable=redefined-outer-name
    16 
    +
    17 
    +import multiprocessing
    18 
    +import os
    19 
    +
    20 
    +import grpc
    21 
    +import pytest
    22 
    +
    23 
    +from buildgrid.client.cas import upload
    24 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    25 
    +
    26 
    +from ..utils.cas import serve_cas, kill_process_tree
    27 
    +
    28 
    +
    29 
    +INTANCES = ['', 'instance']
    30 
    +BLOBS = [(b'',), (b'test-string',), (b'test', b'string')]
    31 
    +MESSAGES = [
    32 
    +    (remote_execution_pb2.Directory(),),
    33 
    +    (remote_execution_pb2.SymlinkNode(name='name', target='target'),),
    34 
    +    (remote_execution_pb2.Action(do_not_cache=True),
    35 
    +     remote_execution_pb2.ActionResult(exit_code=12))
    36 
    +]
    37 
    +DATA_DIR = os.path.join(
    38 
    +    os.path.dirname(os.path.realpath(__file__)), 'data')
    39 
    +FILES = [
    40 
    +    (os.path.join(DATA_DIR, 'void'),),
    41 
    +    (os.path.join(DATA_DIR, 'hello.cc'),),
    42 
    +    (os.path.join(DATA_DIR, 'hello', 'hello.c'),
    43 
    +     os.path.join(DATA_DIR, 'hello', 'hello.h'))]
    44 
    +DIRECTORIES = [
    45 
    +    (remote_execution_pb2.Directory(),),
    46 
    +    (remote_execution_pb2.Directory(
    47 
    +        files=[remote_execution_pb2.FileNode(name='helloc.c'),
    48 
    +               remote_execution_pb2.FileNode(name='helloc.h')]),)]
    49 
    +
    50 
    +
    51 
    +def run_in_subprocess(function, *arguments):
    52 
    +    queue = multiprocessing.Queue()
    53 
    +    # Use subprocess to avoid creation of gRPC threads in main process
    54 
    +    # See https://github.com/grpc/grpc/blob/master/doc/fork_support.md
    55 
    +    process = multiprocessing.Process(target=function,
    56 
    +                                      args=(queue, *arguments))
    57 
    +
    58 
    +    try:
    59 
    +        process.start()
    60 
    +
    61 
    +        result = queue.get()
    62 
    +        process.join()
    63 
    +    except KeyboardInterrupt:
    64 
    +        kill_process_tree(process.pid)
    65 
    +        raise
    66 
    +
    67 
    +    return result
    68 
    +
    69 
    +
    70 
    +@pytest.mark.parametrize('blobs', BLOBS)
    71 
    +@pytest.mark.parametrize('instance', INTANCES)
    72 
    +def test_upload_blob(instance, blobs):
    73 
    +    # Actual test function, to be run in a subprocess:
    74 
    +    def __test_upload_blob(queue, remote, instance, blobs):
    75 
    +        # Open a channel to the remote CAS server:
    76 
    +        channel = grpc.insecure_channel(remote)
    77 
    +
    78 
    +        digests = list()
    79 
    +        with upload(channel, instance) as client:
    80 
    +            if len(blobs) > 1:
    81 
    +                for blob in blobs:
    82 
    +                    digest = client.put_blob(blob, queue=True)
    83 
    +                    digests.append(digest.SerializeToString())
    84 
    +            else:
    85 
    +                digest = client.put_blob(blobs[0], queue=False)
    86 
    +                digests.append(digest.SerializeToString())
    87 
    +
    88 
    +        queue.put(digests)
    89 
    +
    90 
    +    # Start a minimal CAS server in a subprocess:
    91 
    +    with serve_cas([instance]) as server:
    92 
    +        digests = run_in_subprocess(__test_upload_blob,
    93 
    +                                    server.remote, instance, blobs)
    94 
    +
    95 
    +        for blob, digest_blob in zip(blobs, digests):
    96 
    +            digest = remote_execution_pb2.Digest()
    97 
    +            digest.ParseFromString(digest_blob)
    98 
    +
    99 
    +            assert server.has(digest)
    100 
    +            assert server.compare_blobs(digest, blob)
    101 
    +
    102 
    +
    103 
    +@pytest.mark.parametrize('messages', MESSAGES)
    104 
    +@pytest.mark.parametrize('instance', INTANCES)
    105 
    +def test_upload_message(instance, messages):
    106 
    +    # Actual test function, to be run in a subprocess:
    107 
    +    def __test_upload_message(queue, remote, instance, messages):
    108 
    +        # Open a channel to the remote CAS server:
    109 
    +        channel = grpc.insecure_channel(remote)
    110 
    +
    111 
    +        digests = list()
    112 
    +        with upload(channel, instance) as client:
    113 
    +            if len(messages) > 1:
    114 
    +                for message in messages:
    115 
    +                    digest = client.put_message(message, queue=True)
    116 
    +                    digests.append(digest.SerializeToString())
    117 
    +            else:
    118 
    +                digest = client.put_message(messages[0], queue=False)
    119 
    +                digests.append(digest.SerializeToString())
    120 
    +
    121 
    +        queue.put(digests)
    122 
    +
    123 
    +    # Start a minimal CAS server in a subprocess:
    124 
    +    with serve_cas([instance]) as server:
    125 
    +        digests = run_in_subprocess(__test_upload_message,
    126 
    +                                    server.remote, instance, messages)
    127 
    +
    128 
    +        for message, digest_blob in zip(messages, digests):
    129 
    +            digest = remote_execution_pb2.Digest()
    130 
    +            digest.ParseFromString(digest_blob)
    131 
    +
    132 
    +            assert server.has(digest)
    133 
    +            assert server.compare_messages(digest, message)
    134 
    +
    135 
    +
    136 
    +@pytest.mark.parametrize('file_paths', FILES)
    137 
    +@pytest.mark.parametrize('instance', INTANCES)
    138 
    +def test_upload_file(instance, file_paths):
    139 
    +    # Actual test function, to be run in a subprocess:
    140 
    +    def __test_upload_file(queue, remote, instance, file_paths):
    141 
    +        # Open a channel to the remote CAS server:
    142 
    +        channel = grpc.insecure_channel(remote)
    143 
    +
    144 
    +        digests = list()
    145 
    +        with upload(channel, instance) as client:
    146 
    +            if len(file_paths) > 1:
    147 
    +                for file_path in file_paths:
    148 
    +                    digest = client.upload_file(file_path, queue=True)
    149 
    +                    digests.append(digest.SerializeToString())
    150 
    +            else:
    151 
    +                digest = client.upload_file(file_paths[0], queue=False)
    152 
    +                digests.append(digest.SerializeToString())
    153 
    +
    154 
    +        queue.put(digests)
    155 
    +
    156 
    +    # Start a minimal CAS server in a subprocess:
    157 
    +    with serve_cas([instance]) as server:
    158 
    +        digests = run_in_subprocess(__test_upload_file,
    159 
    +                                    server.remote, instance, file_paths)
    160 
    +
    161 
    +        for file_path, digest_blob in zip(file_paths, digests):
    162 
    +            digest = remote_execution_pb2.Digest()
    163 
    +            digest.ParseFromString(digest_blob)
    164 
    +
    165 
    +            assert server.has(digest)
    166 
    +            assert server.compare_files(digest, file_path)
    167 
    +
    168 
    +
    169 
    +@pytest.mark.parametrize('directories', DIRECTORIES)
    170 
    +@pytest.mark.parametrize('instance', INTANCES)
    171 
    +def test_upload_directory(instance, directories):
    172 
    +    # Actual test function, to be run in a subprocess:
    173 
    +    def __test_upload_directory(queue, remote, instance, directories):
    174 
    +        # Open a channel to the remote CAS server:
    175 
    +        channel = grpc.insecure_channel(remote)
    176 
    +
    177 
    +        digests = list()
    178 
    +        with upload(channel, instance) as client:
    179 
    +            if len(directories) > 1:
    180 
    +                for directory in directories:
    181 
    +                    digest = client.upload_directory(directory, queue=True)
    182 
    +                    digests.append(digest.SerializeToString())
    183 
    +            else:
    184 
    +                digest = client.upload_directory(directories[0], queue=False)
    185 
    +                digests.append(digest.SerializeToString())
    186 
    +
    187 
    +        queue.put(digests)
    188 
    +
    189 
    +    # Start a minimal CAS server in a subprocess:
    190 
    +    with serve_cas([instance]) as server:
    191 
    +        digests = run_in_subprocess(__test_upload_directory,
    192 
    +                                    server.remote, instance, directories)
    193 
    +
    194 
    +        for directory, digest_blob in zip(directories, digests):
    195 
    +            digest = remote_execution_pb2.Digest()
    196 
    +            digest.ParseFromString(digest_blob)
    197 
    +
    198 
    +            assert server.has(digest)
    199 
    +            assert server.compare_messages(digest, directory)

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]