GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/gitlab-ci] 2 commits: Adding some simple CI.

Title: GitLab

finnball pushed to branch finn/gitlab-ci at BuildGrid / buildgrid

Commits:

9 changed files:

Changes:

  • .coveragerc
    1 
    +[run]
    2 
    +concurrency = multiprocessing
    3 
    +include =
    4 
    +  */buildgrid/*
    5 
    +
    6 
    +omit =
    7 
    +  # Omit profiling helper module
    8 
    +  # Omit generated code
    9 
    +  */buildgrid/google/*
    10 
    +  */.eggs/*
    11 
    +
    12 
    +[report]
    13 
    +show_missing = True
    14 
    +precision = 2
    15 
    +
    16 
    +[paths]
    17 
    +source =
    18 
    +    buildgrid/
    19 
    +    */site-packages/buildgrid/
    20 
    +    */buildgrid/buildgrid/

  • .gitlab-ci.yml
    1 
    +image: buildstream/buildstream-fedora:master-81-06ae434
    2 
    +
    3 
    +variables:
    4 
    +  BGD: bgd --verbose
    5 
    +
    6 
    +stages:
    7 
    +  - test
    8 
    +  - post
    9 
    +
    10 
    +before_script:
    11 
    +  - export PATH=~/.local/bin:${PATH}
    12 
    +  - pip3 install --user -e .
    13 
    +
    14 
    +tests-fedora:
    15 
    +  stage: test
    16 
    +  variables:
    17 
    +    PYTEST_ADDOPTS: "--color=yes"
    18 
    +  script:
    19 
    +    - yum -y install clang libffi-devel openssl-devel python3-devel
    20 
    +    - python3 setup.py test
    21 
    +    - mkdir -p coverage/
    22 
    +    - cp .coverage.* coverage/coverage."${CI_JOB_NAME}"
    23 
    +  artifacts:
    24 
    +    paths:
    25 
    +    - coverage/
    26 
    +
    27 
    +
    28 
    +tests-dummy-job-fedora:
    29 
    +  stage: test
    30 
    +  script:
    31 
    +    - ${BGD} server start &
    32 
    +    - sleep 1 # Allow server to boot
    33 
    +    - ${BGD} bot --host=0.0.0.0 dummy &
    34 
    +    - ${BGD} execute --host=0.0.0.0 request --wait-for-completion
    35 
    +
    36 
    +coverage:
    37 
    +  stage: post
    38 
    +  coverage: '/TOTAL +\d+ +\d+ +(\d+\.\d+)%/'
    39 
    +  script:
    40 
    +    - mkdir report
    41 
    +    - cd report
    42 
    +    - cp ../coverage/coverage.* .
    43 
    +    - ls coverage.*
    44 
    +    - coverage combine --rcfile=../.coveragerc -a coverage.*
    45 
    +    - coverage report --rcfile=../.coveragerc -m
    46 
    +  dependencies:
    47 
    +  - tests-fedora

  • app/commands/cmd_bot.py
    ... ... @@ -76,10 +76,6 @@ def dummy(context):
    76 76 
         except KeyboardInterrupt:
    77 77 
             pass
    78 78
    79 
    -    except Exception as e:
    80 
    -        context.logger.error(e)
    81 
    -        return
    82 
    -
    83 79 
     @cli.command('buildbox', short_help='Create a bot session with busybox')
    84 80 
     @click.option('--fuse-dir', show_default = True, default=str(PurePath(Path.home(), 'fuse')))
    85 81 
     @click.option('--local-cas', show_default = True, default=str(PurePath(Path.home(), 'cas')))
    ... ... @@ -114,10 +110,6 @@ def _work_buildbox(context, remote, port, server_cert, client_key, client_cert,
    114 110 
         except KeyboardInterrupt:
    115 111 
             pass
    116 112
    117 
    -    except Exception as e:
    118 
    -        context.logger.error(e)
    119 
    -        return
    120 
    -
    121 113 
     async def _work_dummy(context, lease):
    122 114 
         await asyncio.sleep(random.randint(1,5))
    123 115 
         return lease
    ... ... @@ -142,51 +134,46 @@ async def _work_buildbox(context, lease):
    142 134
    143 135 
         stub = bytestream_pb2_grpc.ByteStreamStub(channel)
    144 136
    145 
    -    try:
    146 
    -        remote_command = _fetch_command(context.local_cas, stub, action.command_digest)
    147 
    -        environment = dict((x.name, x.value) for x in remote_command.environment_variables)
    148 
    -        logger.debug("command hash: {}".format(action.command_digest.hash))
    149 
    -        logger.debug("vdir hash: {}".format(action.input_root_digest.hash))
    150 
    -        logger.debug("\n{}".format(' '.join(remote_command.arguments)))
    137 
    +    remote_command = _fetch_command(context.local_cas, stub, action.command_digest)
    138 
    +    environment = dict((x.name, x.value) for x in remote_command.environment_variables)
    139 
    +    logger.debug("command hash: {}".format(action.command_digest.hash))
    140 
    +    logger.debug("vdir hash: {}".format(action.input_root_digest.hash))
    141 
    +    logger.debug("\n{}".format(' '.join(remote_command.arguments)))
    151 142
    152 
    -        command = ['buildbox',
    153 
    -                   '--remote={}'.format('https://{}:{}'.format(context.remote, context.port)),
    154 
    -                   '--server-cert={}'.format(context.server_cert),
    155 
    -                   '--client-key={}'.format(context.client_key),
    156 
    -                   '--client-cert={}'.format(context.client_cert),
    157 
    -                   '--local={}'.format(context.local_cas),
    158 
    -                   '--chdir={}'.format(environment['PWD']),
    159 
    -                   context.fuse_dir,
    160 
    -                   ]
    143 
    +    command = ['buildbox',
    144 
    +               '--remote={}'.format('https://{}:{}'.format(context.remote, context.port)),
    145 
    +               '--server-cert={}'.format(context.server_cert),
    146 
    +               '--client-key={}'.format(context.client_key),
    147 
    +               '--client-cert={}'.format(context.client_cert),
    148 
    +               '--local={}'.format(context.local_cas),
    149 
    +               '--chdir={}'.format(environment['PWD']),
    150 
    +               context.fuse_dir]
    161 151
    162 
    -        command.extend(remote_command.arguments)
    152 
    +    command.extend(remote_command.arguments)
    163 153
    164 
    -        logger.debug(' '.join(command))
    165 
    -        logger.debug("Input root digest:\n{}".format(action.input_root_digest))
    166 
    -        logger.info("Launching process")
    154 
    +    logger.debug(' '.join(command))
    155 
    +    logger.debug("Input root digest:\n{}".format(action.input_root_digest))
    156 
    +    logger.info("Launching process")
    167 157
    168 
    -        proc = subprocess.Popen(command,
    169 
    -                                stdin=subprocess.PIPE,
    170 
    -                                stdout=subprocess.PIPE)
    171 
    -        std_send = action.input_root_digest.SerializeToString()
    172 
    -        std_out, std_error = proc.communicate(std_send)
    158 
    +    proc = subprocess.Popen(command,
    159 
    +                            stdin=subprocess.PIPE,
    160 
    +                            stdout=subprocess.PIPE)
    161 
    +    std_send = action.input_root_digest.SerializeToString()
    162 
    +    std_out, std_error = proc.communicate(std_send)
    173 163
    174 
    -        output_root_digest = remote_execution_pb2.Digest()
    175 
    -        output_root_digest.ParseFromString(std_out)
    176 
    -        logger.debug("Output root digest: {}".format(output_root_digest))
    164 
    +    output_root_digest = remote_execution_pb2.Digest()
    165 
    +    output_root_digest.ParseFromString(std_out)
    166 
    +    logger.debug("Output root digest: {}".format(output_root_digest))
    177 167
    178 
    -        output_file = remote_execution_pb2.OutputDirectory(tree_digest = output_root_digest)
    168 
    +    output_file = remote_execution_pb2.OutputDirectory(tree_digest = output_root_digest)
    179 169
    180 
    -        action_result = remote_execution_pb2.ActionResult()
    181 
    -        action_result.output_directories.extend([output_file])
    170 
    +    action_result = remote_execution_pb2.ActionResult()
    171 
    +    action_result.output_directories.extend([output_file])
    182 172
    183 
    -        action_result_any = any_pb2.Any()
    184 
    -        action_result_any.Pack(action_result)
    173 
    +    action_result_any = any_pb2.Any()
    174 
    +    action_result_any.Pack(action_result)
    185 175
    186 
    -        lease.inline_assignment.CopyFrom(action_result_any)
    187 
    -
    188 
    -    except Exception as e:
    189 
    -        raise Exception(e)
    176 
    +    lease.inline_assignment.CopyFrom(action_result_any)
    190 177
    191 178 
         return lease
    192 179
    ... ... @@ -210,8 +197,5 @@ def _fetch_command(casdir, remote, digest):
    210 197 
             return remote_command
    211 198
    212 199 
     def _file_read(file_path):
    213 
    -    try:
    214 
    -        with open(file_path, 'rb') as f:
    215 
    -            return f.read()
    216 
    -    except Exception as e:
    217 
    -        raise Exception("Error reading: {}. Error: {}".format(file_path, e))
    200 
    +    with open(file_path, 'rb') as f:
    201 
    +        return f.read()

  • app/commands/cmd_execute.py
    ... ... @@ -25,6 +25,8 @@ Request work to be executed and monitor status of jobs.
    25 25 
     import click
    26 26 
     import grpc
    27 27 
     import logging
    28 
    +import sys
    29 
    +import time
    28 30
    29 31 
     from ..cli import pass_context
    30 32
    ... ... @@ -35,19 +37,21 @@ from google.protobuf import any_pb2
    35 37
    36 38 
     @click.group(short_help = "Simple execute client")
    37 39 
     @click.option('--port', default='50051')
    40 
    +@click.option('--host', default='localhost')
    38 41 
     @pass_context
    39 
    -def cli(context, port):
    42 
    +def cli(context, host, port):
    40 43 
         context.logger = logging.getLogger(__name__)
    41 44 
         context.logger.info("Starting on port {}".format(port))
    42 45
    43 
    -    context.channel = grpc.insecure_channel('localhost:{}'.format(port))
    46 
    +    context.channel = grpc.insecure_channel('{}:{}'.format(host, port))
    44 47 
         context.port = port
    45 48
    46 49 
     @cli.command('request', short_help='Send a dummy action')
    47 50 
     @click.option('--number', default=1)
    48 51 
     @click.option('--instance-name', default='testing')
    52 
    +@click.option('--wait-for-completion', is_flag=True)
    49 53 
     @pass_context
    50 
    -def request(context, number, instance_name):
    54 
    +def request(context, number, instance_name, wait_for_completion):
    51 55 
         context.logger.info("Sending execution request...\n")
    52 56 
         stub = remote_execution_pb2_grpc.ExecutionStub(context.channel)
    53 57
    ... ... @@ -64,14 +68,23 @@ def request(context, number, instance_name):
    64 68 
         request = remote_execution_pb2.ExecuteRequest(instance_name = instance_name,
    65 69 
                                                       action = action,
    66 70 
                                                       skip_cache_lookup = True)
    67 
    -    try:
    68 
    -        for i in range(0, number):
    69 
    -            response = stub.Execute(request)
    70 
    -            context.logger.info("Response name: {}".format(response.name))
    71 
    +    for i in range(0, number):
    72 
    +        response = stub.Execute(request)
    73 
    +        context.logger.info("Response name: {}".format(response.name))
    71 74
    72 
    -    except Exception as e:
    73 
    -        context.logger.error(e)
    74 
    -        return
    75 
    +    try:
    76 
    +        while wait_for_completion:
    77 
    +            request = operations_pb2.ListOperationsRequest()
    78 
    +            context.logger.debug('Querying to see if jobs are complete.')
    79 
    +            stub = operations_pb2_grpc.OperationsStub(context.channel)
    80 
    +            response = stub.ListOperations(request)
    81 
    +            if all(operation.done for operation in response.operations):
    82 
    +                context.logger.info('Jobs complete')
    83 
    +                break
    84 
    +            time.sleep(1)
    85 
    +
    86 
    +    except KeyboardInterrupt:
    87 
    +        pass
    75 88
    76 89 
     @cli.command('status', short_help='Get the status of an operation')
    77 90 
     @click.argument('operation-name')
    ... ... @@ -82,13 +95,8 @@ def operation_status(context, operation_name):
    82 95
    83 96 
         request = operations_pb2.GetOperationRequest(name=operation_name)
    84 97
    85 
    -    try:
    86 
    -        response = stub.GetOperation(request)
    87 
    -        _log_operation(context, response)
    88 
    -
    89 
    -    except Exception as e:
    90 
    -        context.logger.error(e)
    91 
    -        return
    98 
    +    response = stub.GetOperation(request)
    99 
    +    _log_operation(context, response)
    92 100
    93 101 
     @cli.command('list', short_help='List operations')
    94 102 
     @pass_context
    ... ... @@ -98,12 +106,7 @@ def list_operations(context):
    98 106
    99 107 
         request = operations_pb2.ListOperationsRequest()
    100 108
    101 
    -    try:
    102 
    -        response = stub.ListOperations(request)
    103 
    -
    104 
    -    except Exception as e:
    105 
    -        context.logger.error(e)
    106 
    -        return
    109 
    +    response = stub.ListOperations(request)
    107 110
    108 111 
         if len(response.operations) < 1:
    109 112 
             context.logger.warning("No operations to list")

  • setup.cfg
    1 
    +[aliases]
    2 
    +test=pytest
    3 
    +
    4 
    +[tool:pytest]
    5 
    +#addopts = --pep8 --pylint
    6 
    +addopts = --verbose --cov=buildgrid --cov-config=.coveragerc
    7 
    +python_files = tests/*.py
    8 
    +pep8ignore =
    9 
    +    * E129
    10 
    +    * E125
    11 
    +    */lib/python3* ALL
    12 
    +    */bin/* ALL
    13 
    +    .eggs/* ALL
    14 
    +    *_pb2.py ALL
    15 
    +    *_pb2_grpc.py ALL
    16 
    +pep8maxlinelength = 119
    \ No newline at end of file

  • setup.py
    ... ... @@ -43,18 +43,22 @@ setup(
    43 43 
             'protobuf',
    44 44 
             'grpcio',
    45 45 
             'Click',
    46 
    -        ],
    47 
    -    extras_require={
    48 
    -        'cas-s3': ['boto3', 'botocore'],
    49 
    -        ],
    50 
    -    tests_require=[
    51 
    -        'pytest',
    52 46 
             'boto3',
    53 47 
             'botocore',
    54 
    -        'moto',
    55 
    -        ],
    48 
    +    ],
    56 49 
         entry_points='''
    57 50 
         [console_scripts]
    58 51 
         bgd=app:cli
    59 52 
         ''',
    53 
    +    setup_requires=['pytest-runner'],
    54 
    +    tests_require=['pep8',
    55 
    +                   'boto3',
    56 
    +                   'botocore',
    57 
    +                   'moto',
    58 
    +                   'coverage == 4.4.0',
    59 
    +                   'pytest-cov >= 2.5.0',
    60 
    +                   'pytest-pep8',
    61 
    +                   'pytest-pylint',
    62 
    +                   'pytest >= 3.1.0',
    63 
    +                   'pylint >= 1.8 , < 2'],
    60 64 
     )

  • test/cas/__init__.py → tests/cas/__init__.py

  • test/cas/test_services.py → tests/cas/test_services.py
    ... ... @@ -70,7 +70,7 @@ def raise_mock_exception(*args, **kwargs):
    70 70 
         raise MockException()
    71 71
    72 72
    73 
    -test_strings = [b"", b"hij", b"testing!" * 1000000]
    73 
    +test_strings = [b"", b"hij"]
    74 74 
     instances = ["", "test_inst"]
    75 75
    76 76
    ... ... @@ -91,6 +91,24 @@ def test_bytestream_read(data_to_read, instance):
    91 91 
         assert data == data_to_read
    92 92
    93 93
    94 
    +@pytest.mark.parametrize("instance", instances)
    95 
    +def test_bytestream_read_many(instance):
    96 
    +    data_to_read = b"testing" * 10000
    97 
    +
    98 
    +    storage = SimpleStorage([b"abc", b"defg", data_to_read])
    99 
    +    servicer = ByteStreamService(storage)
    100 
    +
    101 
    +    request = bytestream_pb2.ReadRequest()
    102 
    +    if instance != "":
    103 
    +        request.resource_name = instance + "/"
    104 
    +    request.resource_name += f"blobs/{HASH(data_to_read).hexdigest()}/{len(data_to_read)}"
    105 
    +
    106 
    +    data = b""
    107 
    +    for response in servicer.Read(request, None):
    108 
    +        data += response.data
    109 
    +    assert data == data_to_read
    110 
    +
    111 
    +
    94 112 
     @pytest.mark.parametrize("instance", instances)
    95 113 
     @pytest.mark.parametrize("extra_data", ["", "/", "/extra/data"])
    96 114 
     def test_bytestream_write(instance, extra_data):

  • test/cas/test_storage.py → tests/cas/test_storage.py

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]