GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/144-jwt-authentication] 17 commits: job.py: Expose its ActionResult origin

Title: GitLab

Martin Blanchard pushed to branch mablanch/144-jwt-authentication at BuildGrid / buildgrid

Commits:

30 changed files:

Changes:

  • .gitlab-ci.yml
    ... ... @@ -20,6 +20,7 @@ before_script:
    20 20 
       variables:
    21 21 
         PYTEST_ADDOPTS: "--color=yes"
    22 22 
       script:
    23 
    +    - python3 -m pip install --user --editable ".[auth]"
    23 24 
         - python3 setup.py test
    24 25 
         - mkdir -p coverage/
    25 26 
         - cp .coverage coverage/coverage."${CI_JOB_NAME}"

  • .pylintrc
    ... ... @@ -186,7 +186,8 @@ ignore-on-opaque-inference=yes
    186 186 
     # qualified names.
    187 187 
     ignored-classes=google.protobuf.any_pb2.Any,
    188 188 
                     google.protobuf.duration_pb2.Duration,
    189 
    -                google.protobuf.timestamp_pb2.Timestamp
    189 
    +                google.protobuf.timestamp_pb2.Timestamp,
    190 
    +                jwt
    190 191
    191 192 
     # List of module names for which member attributes should not be checked
    192 193 
     # (useful for modules/projects where namespaces are manipulated during runtime
    ... ... @@ -462,6 +463,7 @@ known-third-party=boto3,
    462 463 
                       google,
    463 464 
                       grpc,
    464 465 
                       janus,
    466 
    +                  jwt,
    465 467 
                       moto,
    466 468 
                       yaml
    467 469

  • buildgrid/_app/commands/cmd_bot.py
    ... ... @@ -22,16 +22,15 @@ Create a bot interface and request work
    22 22
    23 23 
     from pathlib import Path, PurePath
    24 24 
     import sys
    25 
    -from urllib.parse import urlparse
    26 25
    27 26 
     import click
    28 
    -import grpc
    29 27
    30 28 
     from buildgrid.bot import bot, interface, session
    31 29 
     from buildgrid.bot.hardware.interface import HardwareInterface
    32 30 
     from buildgrid.bot.hardware.device import Device
    33 31 
     from buildgrid.bot.hardware.worker import Worker
    34 
    -
    32 
    +from buildgrid.client.authentication import setup_channel
    33 
    +from buildgrid._exceptions import InvalidArgumentError
    35 34
    36 35 
     from ..bots import buildbox, dummy, host
    37 36 
     from ..cli import pass_context, setup_logging
    ... ... @@ -40,20 +39,22 @@ from ..cli import pass_context, setup_logging
    40 39 
     @click.group(name='bot', short_help="Create and register bot clients.")
    41 40 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    42 41 
                   help="Remote execution server's URL (port defaults to 50051 if not specified).")
    42 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    43 
    +              help="Authorization token for the remote.")
    43 44 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    -              help="Private client key for TLS (PEM-encoded)")
    45 
    +              help="Private client key for TLS (PEM-encoded).")
    45 46 
     @click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    46 
    -              help="Public client certificate for TLS (PEM-encoded)")
    47 
    +              help="Public client certificate for TLS (PEM-encoded).")
    47 48 
     @click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    48 
    -              help="Public server certificate for TLS (PEM-encoded)")
    49 
    +              help="Public server certificate for TLS (PEM-encoded).")
    49 50 
     @click.option('--remote-cas', type=click.STRING, default=None, show_default=True,
    50 51 
                   help="Remote CAS server's URL (port defaults to 11001 if not specified).")
    51 52 
     @click.option('--cas-client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    52 
    -              help="Private CAS client key for TLS (PEM-encoded)")
    53 
    +              help="Private CAS client key for TLS (PEM-encoded).")
    53 54 
     @click.option('--cas-client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    54 
    -              help="Public CAS client certificate for TLS (PEM-encoded)")
    55 
    +              help="Public CAS client certificate for TLS (PEM-encoded).")
    55 56 
     @click.option('--cas-server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    56 
    -              help="Public CAS server certificate for TLS (PEM-encoded)")
    57 
    +              help="Public CAS server certificate for TLS (PEM-encoded).")
    57 58 
     @click.option('--update-period', type=click.FLOAT, default=0.5, show_default=True,
    58 59 
                   help="Time period for bot updates to the server in seconds.")
    59 60 
     @click.option('--parent', type=click.STRING, default='main', show_default=True,
    ... ... @@ -61,69 +62,31 @@ from ..cli import pass_context, setup_logging
    61 62 
     @click.option('-v', '--verbose', count=True,
    62 63 
                   help='Increase log verbosity level.')
    63 64 
     @pass_context
    64 
    -def cli(context, parent, update_period, remote, client_key, client_cert, server_cert,
    65 
    +def cli(context, parent, update_period, remote, auth_token, client_key, client_cert, server_cert,
    65 66 
             remote_cas, cas_client_key, cas_client_cert, cas_server_cert, verbose):
    66 67 
         setup_logging(verbosity=verbose)
    67 68 
         # Setup the remote execution server channel:
    68 
    -    url = urlparse(remote)
    69 
    -
    70 
    -    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    71 
    -    context.remote_url = remote
    72 
    -    context.update_period = update_period
    73 
    -    context.parent = parent
    74 
    -
    75 
    -    if url.scheme == 'http':
    76 
    -        context.channel = grpc.insecure_channel(context.remote)
    77 
    -
    78 
    -        context.client_key = None
    79 
    -        context.client_cert = None
    80 
    -        context.server_cert = None
    81 
    -    else:
    82 
    -        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    83 
    -        if not credentials:
    84 
    -            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    85 
    -            sys.exit(-1)
    86 
    -
    87 
    -        context.channel = grpc.secure_channel(context.remote, credentials)
    88 
    -
    89 
    -        context.client_key = credentials.client_key
    90 
    -        context.client_cert = credentials.client_cert
    91 
    -        context.server_cert = credentials.server_cert
    92 
    -
    93 
    -    # Setup the remote CAS server channel, if separated:
    94 
    -    if remote_cas is not None and remote_cas != remote:
    95 
    -        cas_url = urlparse(remote_cas)
    96 
    -
    97 
    -        context.remote_cas = '{}:{}'.format(cas_url.hostname, cas_url.port or 11001)
    98 
    -        context.remote_cas_url = remote_cas
    69 
    +    try:
    70 
    +        context.channel, details = setup_channel(remote, auth_token=auth_token, server_cert=server_cert,
    71 
    +                                                 client_key=client_key, client_cert=client_cert)
    99 72
    100 
    -        if cas_url.scheme == 'http':
    101 
    -            context.cas_channel = grpc.insecure_channel(context.remote_cas)
    73 
    +        if remote_cas is not None and remote_cas != remote:
    74 
    +            context.cas_channel, details = setup_channel(remote_cas, server_cert=cas_server_cert,
    75 
    +                                                         client_key=cas_client_key, client_cert=cas_client_cert)
    76 
    +            context.remote_cas_url = remote_cas
    102 77
    103 
    -            context.cas_client_key = None
    104 
    -            context.cas_client_cert = None
    105 
    -            context.cas_server_cert = None
    106 78 
             else:
    107 
    -            cas_credentials = context.load_client_credentials(cas_client_key, cas_client_cert, cas_server_cert)
    108 
    -            if not cas_credentials:
    109 
    -                click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    110 
    -                sys.exit(-1)
    111 
    -
    112 
    -            context.cas_channel = grpc.secure_channel(context.remote_cas, cas_credentials)
    79 
    +            context.cas_channel = context.channel
    80 
    +            context.remote_cas_url = remote
    113 81
    114 
    -            context.cas_client_key = cas_credentials.client_key
    115 
    -            context.cas_client_cert = cas_credentials.client_cert
    116 
    -            context.cas_server_cert = cas_credentials.server_cert
    82 
    +        context.cas_client_key, context.cas_client_cert, context.cas_server_cert = details
    117 83
    118 
    -    else:
    119 
    -        context.remote_cas = context.remote
    120 
    -        context.remote_cas_url = remote
    84 
    +    except InvalidArgumentError as e:
    85 
    +        click.echo("Error: {}.".format(e), err=True)
    86 
    +        sys.exit(-1)
    121 87
    122 
    -        context.cas_channel = context.channel
    123 
    -
    124 
    -        context.cas_client_key = context.client_key
    125 
    -        context.cas_client_cert = context.client_cert
    126 
    -        context.cas_server_cert = context.server_cert
    88 
    +    context.update_period = update_period
    89 
    +    context.parent = parent
    127 90
    128 91 
         bot_interface = interface.BotInterface(context.channel)
    129 92

  • buildgrid/_app/commands/cmd_capabilities.py
    ... ... @@ -14,12 +14,13 @@
    14 14
    15 15
    16 16 
     import sys
    17 
    -from urllib.parse import urlparse
    18 17
    19 18 
     import click
    20 
    -import grpc
    19 
    +from google.protobuf import json_format
    21 20
    21 
    +from buildgrid.client.authentication import setup_channel
    22 22 
     from buildgrid.client.capabilities import CapabilitiesInterface
    23 
    +from buildgrid._exceptions import InvalidArgumentError
    23 24
    24 25 
     from ..cli import pass_context
    25 26
    ... ... @@ -27,32 +28,30 @@ from ..cli import pass_context
    27 28 
     @click.command(name='capabilities', short_help="Capabilities service.")
    28 29 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    29 30 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    31 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    32 
    +              help="Authorization token for the remote.")
    30 33 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    31 
    -              help="Private client key for TLS (PEM-encoded)")
    34 
    +              help="Private client key for TLS (PEM-encoded).")
    32 35 
     @click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    33 
    -              help="Public client certificate for TLS (PEM-encoded)")
    36 
    +              help="Public client certificate for TLS (PEM-encoded).")
    34 37 
     @click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    35 
    -              help="Public server certificate for TLS (PEM-encoded)")
    38 
    +              help="Public server certificate for TLS (PEM-encoded).")
    36 39 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    37 40 
                   help="Targeted farm instance name.")
    38 41 
     @pass_context
    39 
    -def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    40 
    -    click.echo("Getting capabilities...")
    41 
    -    url = urlparse(remote)
    42 
    -
    43 
    -    remote = '{}:{}'.format(url.hostname, url.port or 50051)
    44 
    -    instance_name = instance_name
    45 
    -
    46 
    -    if url.scheme == 'http':
    47 
    -        channel = grpc.insecure_channel(remote)
    48 
    -    else:
    49 
    -        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    50 
    -        if not credentials:
    51 
    -            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    52 
    -            sys.exit(-1)
    53 
    -
    54 
    -        channel = grpc.secure_channel(remote, credentials)
    55 
    -
    56 
    -    interface = CapabilitiesInterface(channel)
    57 
    -    response = interface.get_capabilities(instance_name)
    58 
    -    click.echo(response)
    42 
    +def cli(context, remote, instance_name, auth_token, client_key, client_cert, server_cert):
    43 
    +    """Entry point for the bgd-capabilities CLI command group."""
    44 
    +    try:
    45 
    +        context.channel, _ = setup_channel(remote, auth_token=auth_token,
    46 
    +                                           client_key=client_key, client_cert=client_cert)
    47 
    +
    48 
    +    except InvalidArgumentError as e:
    49 
    +        click.echo("Error: {}.".format(e), err=True)
    50 
    +        sys.exit(-1)
    51 
    +
    52 
    +    context.instance_name = instance_name
    53 
    +
    54 
    +    interface = CapabilitiesInterface(context.channel)
    55 
    +    response = interface.get_capabilities(context.instance_name)
    56 
    +
    57 
    +    click.echo(json_format.MessageToJson(response))

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -22,12 +22,12 @@ Request work to be executed and monitor status of jobs.
    22 22
    23 23 
     import os
    24 24 
     import sys
    25 
    -from urllib.parse import urlparse
    26 25
    27 26 
     import click
    28 
    -import grpc
    29 27
    28 
    +from buildgrid.client.authentication import setup_channel
    30 29 
     from buildgrid.client.cas import download, upload
    30 
    +from buildgrid._exceptions import InvalidArgumentError
    31 31 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    32 32 
     from buildgrid.utils import create_digest, merkle_tree_maker, read_file
    33 33
    ... ... @@ -37,32 +37,28 @@ from ..cli import pass_context
    37 37 
     @click.group(name='cas', short_help="Interact with the CAS server.")
    38 38 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    39 39 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    40 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    41 
    +              help="Authorization token for the remote.")
    40 42 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    41 
    -              help="Private client key for TLS (PEM-encoded)")
    43 
    +              help="Private client key for TLS (PEM-encoded).")
    42 44 
     @click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    43 
    -              help="Public client certificate for TLS (PEM-encoded)")
    45 
    +              help="Public client certificate for TLS (PEM-encoded).")
    44 46 
     @click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    45 47 
                   help="Public server certificate for TLS (PEM-encoded)")
    46 48 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    47 49 
                   help="Targeted farm instance name.")
    48 50 
     @pass_context
    49 
    -def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    50 
    -    url = urlparse(remote)
    51 
    +def cli(context, remote, instance_name, auth_token, client_key, client_cert, server_cert):
    52 
    +    """Entry point for the bgd-cas CLI command group."""
    53 
    +    try:
    54 
    +        context.channel, _ = setup_channel(remote, auth_token=auth_token,
    55 
    +                                           client_key=client_key, client_cert=client_cert)
    51 56
    52 
    -    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    53 
    -    context.instance_name = instance_name
    54 
    -
    55 
    -    if url.scheme == 'http':
    56 
    -        context.channel = grpc.insecure_channel(context.remote)
    57 
    -    else:
    58 
    -        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    59 
    -        if not credentials:
    60 
    -            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    61 
    -            sys.exit(-1)
    57 
    +    except InvalidArgumentError as e:
    58 
    +        click.echo("Error: {}.".format(e), err=True)
    59 
    +        sys.exit(-1)
    62 60
    63 
    -        context.channel = grpc.secure_channel(context.remote, credentials)
    64 
    -
    65 
    -    click.echo("Starting for remote=[{}]".format(context.remote))
    61 
    +    context.instance_name = instance_name
    66 62
    67 63
    68 64 
     @cli.command('upload-dummy', short_help="Upload a dummy action. Should be used with `execute dummy-request`")

  • buildgrid/_app/commands/cmd_execute.py
    ... ... @@ -23,12 +23,12 @@ Request work to be executed and monitor status of jobs.
    23 23 
     import os
    24 24 
     import stat
    25 25 
     import sys
    26 
    -from urllib.parse import urlparse
    27 26
    28 27 
     import click
    29 
    -import grpc
    30 28
    29 
    +from buildgrid.client.authentication import setup_channel
    31 30 
     from buildgrid.client.cas import download, upload
    31 
    +from buildgrid._exceptions import InvalidArgumentError
    32 32 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    33 33 
     from buildgrid.utils import create_digest
    34 34
    ... ... @@ -38,32 +38,28 @@ from ..cli import pass_context
    38 38 
     @click.group(name='execute', short_help="Execute simple operations.")
    39 39 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    40 40 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    41 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Authorization token for the remote.")
    41 43 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    -              help="Private client key for TLS (PEM-encoded)")
    44 
    +              help="Private client key for TLS (PEM-encoded).")
    43 45 
     @click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    44 
    -              help="Public client certificate for TLS (PEM-encoded)")
    46 
    +              help="Public client certificate for TLS (PEM-encoded).")
    45 47 
     @click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    46 
    -              help="Public server certificate for TLS (PEM-encoded)")
    48 
    +              help="Public server certificate for TLS (PEM-encoded).")
    47 49 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    48 50 
                   help="Targeted farm instance name.")
    49 51 
     @pass_context
    50 
    -def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    51 
    -    url = urlparse(remote)
    52 
    +def cli(context, remote, instance_name, auth_token, client_key, client_cert, server_cert):
    53 
    +    """Entry point for the bgd-execute CLI command group."""
    54 
    +    try:
    55 
    +        context.channel, _ = setup_channel(remote, auth_token=auth_token,
    56 
    +                                           client_key=client_key, client_cert=client_cert)
    52 57
    53 
    -    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    54 
    -    context.instance_name = instance_name
    55 
    -
    56 
    -    if url.scheme == 'http':
    57 
    -        context.channel = grpc.insecure_channel(context.remote)
    58 
    -    else:
    59 
    -        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    60 
    -        if not credentials:
    61 
    -            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    62 
    -            sys.exit(-1)
    58 
    +    except InvalidArgumentError as e:
    59 
    +        click.echo("Error: {}.".format(e), err=True)
    60 
    +        sys.exit(-1)
    63 61
    64 
    -        context.channel = grpc.secure_channel(context.remote, credentials)
    65 
    -
    66 
    -    click.echo("Starting for remote=[{}]".format(context.remote))
    62 
    +    context.instance_name = instance_name
    67 63
    68 64
    69 65 
     @cli.command('request-dummy', short_help="Send a dummy action.")

  • buildgrid/_app/commands/cmd_operation.py
    ... ... @@ -22,15 +22,15 @@ Check the status of operations
    22 22
    23 23 
     from collections import OrderedDict
    24 24 
     from operator import attrgetter
    25 
    -from urllib.parse import urlparse
    26 25 
     import sys
    27 26 
     from textwrap import indent
    28 27
    29 28 
     import click
    30 29 
     from google.protobuf import json_format
    31 
    -import grpc
    32 30
    31 
    +from buildgrid.client.authentication import setup_channel
    33 32 
     from buildgrid._enums import OperationStage
    33 
    +from buildgrid._exceptions import InvalidArgumentError
    34 34 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    35 35 
     from buildgrid._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    36 36 
     from buildgrid._protos.google.rpc import code_pb2
    ... ... @@ -41,32 +41,28 @@ from ..cli import pass_context
    41 41 
     @click.group(name='operation', short_help="Long running operations commands.")
    42 42 
     @click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    43 43 
                   help="Remote execution server's URL (port defaults to 50051 if no specified).")
    44 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    45 
    +              help="Authorization token for the remote.")
    44 46 
     @click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    45 
    -              help="Private client key for TLS (PEM-encoded)")
    47 
    +              help="Private client key for TLS (PEM-encoded).")
    46 48 
     @click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    47 
    -              help="Public client certificate for TLS (PEM-encoded)")
    49 
    +              help="Public client certificate for TLS (PEM-encoded).")
    48 50 
     @click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    49 
    -              help="Public server certificate for TLS (PEM-encoded)")
    51 
    +              help="Public server certificate for TLS (PEM-encoded).")
    50 52 
     @click.option('--instance-name', type=click.STRING, default='main', show_default=True,
    51 53 
                   help="Targeted farm instance name.")
    52 54 
     @pass_context
    53 
    -def cli(context, remote, instance_name, client_key, client_cert, server_cert):
    54 
    -    url = urlparse(remote)
    55 
    +def cli(context, remote, instance_name, auth_token, client_key, client_cert, server_cert):
    56 
    +    """Entry point for the bgd-operation CLI command group."""
    57 
    +    try:
    58 
    +        context.channel, _ = setup_channel(remote, auth_token=auth_token,
    59 
    +                                           client_key=client_key, client_cert=client_cert)
    55 60
    56 
    -    context.remote = '{}:{}'.format(url.hostname, url.port or 50051)
    57 
    -    context.instance_name = instance_name
    58 
    -
    59 
    -    if url.scheme == 'http':
    60 
    -        context.channel = grpc.insecure_channel(context.remote)
    61 
    -    else:
    62 
    -        credentials = context.load_client_credentials(client_key, client_cert, server_cert)
    63 
    -        if not credentials:
    64 
    -            click.echo("ERROR: no TLS keys were specified and no defaults could be found.", err=True)
    65 
    -            sys.exit(-1)
    61 
    +    except InvalidArgumentError as e:
    62 
    +        click.echo("Error: {}.".format(e), err=True)
    63 
    +        sys.exit(-1)
    66 64
    67 
    -        context.channel = grpc.secure_channel(context.remote, credentials)
    68 
    -
    69 
    -    click.echo("Starting for remote=[{}]".format(context.remote))
    65 
    +    context.instance_name = instance_name
    70 66
    71 67
    72 68 
     def _print_operation_status(operation, print_details=False):

  • buildgrid/client/authentication.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +import base64
    17 
    +from collections import namedtuple
    18 
    +from urllib.parse import urlparse
    19 
    +import os
    20 
    +
    21 
    +import grpc
    22 
    +
    23 
    +from buildgrid._exceptions import InvalidArgumentError
    24 
    +from buildgrid.utils import read_file
    25 
    +
    26 
    +
    27 
    +def load_tls_channel_credentials(client_key=None, client_cert=None, server_cert=None):
    28 
    +    """Looks-up and loads TLS gRPC client channel credentials.
    29 
    +
    30 
    +    Args:
    31 
    +        client_key(str, optional): Client certificate chain file path.
    32 
    +        client_cert(str, optional): Client private key file path.
    33 
    +        server_cert(str, optional): Serve root certificate file path.
    34 
    +
    35 
    +    Returns:
    36 
    +        ChannelCredentials: Credentials to be used for a TLS-encrypted gRPC
    37 
    +            client channel.
    38 
    +    """
    39 
    +    if server_cert and os.path.exists(server_cert):
    40 
    +        server_cert_pem = read_file(server_cert)
    41 
    +    else:
    42 
    +        server_cert_pem = None
    43 
    +        server_cert = None
    44 
    +
    45 
    +    if client_key and os.path.exists(client_key):
    46 
    +        client_key_pem = read_file(client_key)
    47 
    +    else:
    48 
    +        client_key_pem = None
    49 
    +        client_key = None
    50 
    +
    51 
    +    if client_key_pem and client_cert and os.path.exists(client_cert):
    52 
    +        client_cert_pem = read_file(client_cert)
    53 
    +    else:
    54 
    +        client_cert_pem = None
    55 
    +        client_cert = None
    56 
    +
    57 
    +    credentials = grpc.ssl_channel_credentials(root_certificates=server_cert_pem,
    58 
    +                                               private_key=client_key_pem,
    59 
    +                                               certificate_chain=client_cert_pem)
    60 
    +
    61 
    +    return credentials, (client_key, client_cert, server_cert,)
    62 
    +
    63 
    +
    64 
    +def load_channel_authorization_token(auth_token=None):
    65 
    +    """Looks-up and loads client authorization token.
    66 
    +
    67 
    +    Args:
    68 
    +        auth_token (str, optional): Token file path.
    69 
    +
    70 
    +    Returns:
    71 
    +        str: Encoded token string.
    72 
    +    """
    73 
    +    if auth_token and os.path.exists(auth_token):
    74 
    +        return read_file(auth_token).decode()
    75 
    +
    76 
    +    # TODO: Try loading the token from a default location?
    77 
    +
    78 
    +    return None
    79 
    +
    80 
    +
    81 
    +def setup_channel(remote_url, auth_token=None,
    82 
    +                  client_key=None, client_cert=None, server_cert=None):
    83 
    +    """Creates a new gRPC client communication chanel.
    84 
    +
    85 
    +    If `remote_url` does not specifies a port number, defaults 50051.
    86 
    +
    87 
    +    Args:
    88 
    +        remote_url (str): URL for the remote, including port and protocol.
    89 
    +        auth_token (str): Authorization token file path.
    90 
    +        server_cert(str): TLS certificate chain file path.
    91 
    +        client_key(str): TLS root certificate file path.
    92 
    +        client_cert(str): TLS private key file path.
    93 
    +
    94 
    +    Returns:
    95 
    +        Channel: Client Channel to be used in order to access the server
    96 
    +            at `remote_url`.
    97 
    +
    98 
    +    Raises:
    99 
    +        InvalidArgumentError: On any input parsing error.
    100 
    +    """
    101 
    +    url = urlparse(remote_url)
    102 
    +    remote = '{}:{}'.format(url.hostname, url.port or 50051)
    103 
    +    details = None, None, None
    104 
    +
    105 
    +    if url.scheme == 'http':
    106 
    +        channel = grpc.insecure_channel(remote)
    107 
    +
    108 
    +    elif url.scheme == 'https':
    109 
    +        credentials, details = load_tls_channel_credentials(client_key, client_cert, server_cert)
    110 
    +        if not credentials:
    111 
    +            raise InvalidArgumentError("Given TLS details (or defaults) could be loaded")
    112 
    +
    113 
    +        channel = grpc.secure_channel(remote, credentials)
    114 
    +
    115 
    +    else:
    116 
    +        raise InvalidArgumentError("Given remote does not specify a protocol")
    117 
    +
    118 
    +    if auth_token is not None:
    119 
    +        token = load_channel_authorization_token(auth_token)
    120 
    +        if not token:
    121 
    +            raise InvalidArgumentError("Given authorization token could be loaded")
    122 
    +
    123 
    +        interpector = AuthMetadataClientInterceptor(auth_token=token)
    124 
    +        channel = grpc.intercept_channel(channel, interpector)
    125 
    +
    126 
    +    return channel, details
    127 
    +
    128 
    +
    129 
    +class AuthMetadataClientInterceptor(
    130 
    +        grpc.UnaryUnaryClientInterceptor, grpc.UnaryStreamClientInterceptor,
    131 
    +        grpc.StreamUnaryClientInterceptor, grpc.StreamStreamClientInterceptor):
    132 
    +
    133 
    +    def __init__(self, auth_token=None, auth_secret=None):
    134 
    +        """Initialises a new :class:`AuthMetadataClientInterceptor`.
    135 
    +
    136 
    +        Important:
    137 
    +            One of `auth_token` or `auth_secret` must be provided.
    138 
    +
    139 
    +        Args:
    140 
    +            auth_token (str, optional): Authorization token as a string.
    141 
    +            auth_secret (str, optional): Authorization secret as a string.
    142 
    +
    143 
    +        Raises:
    144 
    +            InvalidArgumentError: If neither `auth_token` or `auth_secret` are
    145 
    +                provided.
    146 
    +        """
    147 
    +        if auth_token:
    148 
    +            self.__secret = auth_token.strip()
    149 
    +
    150 
    +        elif auth_secret:
    151 
    +            self.__secret = base64.b64encode(auth_secret.strip())
    152 
    +
    153 
    +        else:
    154 
    +            raise InvalidArgumentError("A secret or token must be provided")
    155 
    +
    156 
    +        self.__header_field_name = 'authorization'
    157 
    +        self.__header_field_value = 'Bearer {}'.format(self.__secret)
    158 
    +
    159 
    +    # --- Public API ---
    160 
    +
    161 
    +    def intercept_unary_unary(self, continuation, client_call_details, request):
    162 
    +        new_details = self._amend_call_details(client_call_details)
    163 
    +
    164 
    +        return continuation(new_details, request)
    165 
    +
    166 
    +    def intercept_unary_stream(self, continuation, client_call_details, request):
    167 
    +        new_details = self._amend_call_details(client_call_details)
    168 
    +
    169 
    +        return continuation(new_details, request)
    170 
    +
    171 
    +    def intercept_stream_unary(self, continuation, client_call_details, request_iterator):
    172 
    +        new_details = self._amend_call_details(client_call_details)
    173 
    +
    174 
    +        return continuation(new_details, request_iterator)
    175 
    +
    176 
    +    def intercept_stream_stream(self, continuation, client_call_details, request_iterator):
    177 
    +        new_details = self._amend_call_details(client_call_details)
    178 
    +
    179 
    +        return continuation(new_details, request_iterator)
    180 
    +
    181 
    +    # --- Private API ---
    182 
    +
    183 
    +    def _amend_call_details(self, client_call_details):
    184 
    +        """Appends an authorization field to given client call details."""
    185 
    +        if client_call_details.metadata is not None:
    186 
    +            new_metadata = list(client_call_details.metadata)
    187 
    +        else:
    188 
    +            new_metadata = []
    189 
    +
    190 
    +        new_metadata.append((self.__header_field_name, self.__header_field_value,))
    191 
    +
    192 
    +        class _ClientCallDetails(
    193 
    +                namedtuple('_ClientCallDetails',
    194 
    +                           ('method', 'timeout', 'credentials', 'metadata')),
    195 
    +                grpc.ClientCallDetails):
    196 
    +            pass
    197 
    +
    198 
    +        return _ClientCallDetails(client_call_details.method,
    199 
    +                                  client_call_details.timeout,
    200 
    +                                  client_call_details.credentials,
    201 
    +                                  new_metadata)

  • buildgrid/server/_authentication.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +from datetime import datetime
    17 
    +from enum import Enum
    18 
    +import logging
    19 
    +
    20 
    +import grpc
    21 
    +
    22 
    +from buildgrid._exceptions import InvalidArgumentError
    23 
    +
    24 
    +
    25 
    +try:
    26 
    +    import jwt
    27 
    +except ImportError:
    28 
    +    HAVE_JWT = False
    29 
    +else:
    30 
    +    HAVE_JWT = True
    31 
    +
    32 
    +
    33 
    +class AuthMetadataMethod(Enum):
    34 
    +    # No authentication:
    35 
    +    NONE = 'none'
    36 
    +    # JWT based authentication:
    37 
    +    JWT = 'jwt'
    38 
    +
    39 
    +
    40 
    +class AuthMetadataAlgorithm(Enum):
    41 
    +    # No encryption involved:
    42 
    +    UNSPECIFIED = 'unspecified'
    43 
    +    # JWT related algorithms:
    44 
    +    JWT_ES256 = 'es256'  # ECDSA signature algorithm using SHA-256 hash algorithm
    45 
    +    JWT_ES384 = 'es384'  # ECDSA signature algorithm using SHA-384 hash algorithm
    46 
    +    JWT_ES512 = 'es512'  # ECDSA signature algorithm using SHA-512 hash algorithm
    47 
    +    JWT_HS256 = 'hs256'  # HMAC using SHA-256 hash algorithm
    48 
    +    JWT_HS384 = 'hs384'  # HMAC using SHA-384 hash algorithm
    49 
    +    JWT_HS512 = 'hs512'  # HMAC using SHA-512 hash algorithm
    50 
    +    JWT_PS256 = 'ps256'  # RSASSA-PSS using SHA-256 and MGF1 padding with SHA-256
    51 
    +    JWT_PS384 = 'ps384'  # RSASSA-PSS signature using SHA-384 and MGF1 padding with SHA-384
    52 
    +    JWT_PS512 = 'ps512'  # RSASSA-PSS signature using SHA-512 and MGF1 padding with SHA-512
    53 
    +    JWT_RS256 = 'rs256'  # RSASSA-PKCS1-v1_5 signature algorithm using SHA-256 hash algorithm
    54 
    +    JWT_RS384 = 'rs384'  # RSASSA-PKCS1-v1_5 signature algorithm using SHA-384 hash algorithm
    55 
    +    JWT_RS512 = 'rs512'  # RSASSA-PKCS1-v1_5 signature algorithm using SHA-512 hash algorithm
    56 
    +
    57 
    +
    58 
    +class _InvalidTokenError(Exception):
    59 
    +    pass
    60 
    +
    61 
    +
    62 
    +class _ExpiredTokenError(Exception):
    63 
    +    pass
    64 
    +
    65 
    +
    66 
    +class _UnboundedTokenError(Exception):
    67 
    +    pass
    68 
    +
    69 
    +
    70 
    +class AuthMetadataServerInterceptor(grpc.ServerInterceptor):
    71 
    +
    72 
    +    __auth_errors = {
    73 
    +        'missing-bearer': 'Missing authentication header field',
    74 
    +        'invalid-bearer': 'Invalid authentication header field',
    75 
    +        'invalid-token': 'Invalid authentication token',
    76 
    +        'expired-token': 'Expired authentication token',
    77 
    +        'unbounded-token': 'Unbounded authentication token',
    78 
    +    }
    79 
    +
    80 
    +    def __init__(self, method, secret=None, algorithm=AuthMetadataAlgorithm.UNSPECIFIED):
    81 
    +        """Initialises a new :class:`AuthMetadataServerInterceptor`.
    82 
    +
    83 
    +        Args:
    84 
    +            method (AuthMetadataMethod): Type of authorization method.
    85 
    +            secret (str): The secret or key to be used for validating request,
    86 
    +                depending on `method`. Defaults to ``None``.
    87 
    +            algorithm (AuthMetadataAlgorithm): The crytographic algorithm used
    88 
    +                to encode `secret`. Defaults to ``UNSPECIFIED``.
    89 
    +
    90 
    +        Raises:
    91 
    +            InvalidArgumentError: If `method` is not supported or if `algorithm`
    92 
    +                is not supported for the given `method`.
    93 
    +        """
    94 
    +        self.__logger = logging.getLogger(__name__)
    95 
    +
    96 
    +        self.__bearer_cache = {}
    97 
    +        self.__terminators = {}
    98 
    +        self.__validator = None
    99 
    +        self.__secret = secret
    100 
    +
    101 
    +        self._method = method
    102 
    +        self._algorithm = algorithm
    103 
    +
    104 
    +        if self._method == AuthMetadataMethod.JWT:
    105 
    +            self._check_jwt_support(self._algorithm)
    106 
    +            self.__validator = self._validate_jwt_token
    107 
    +
    108 
    +        for code, message in self.__auth_errors.items():
    109 
    +            self.__terminators[code] = _unary_unary_rpc_terminator(message)
    110 
    +
    111 
    +    # --- Public API ---
    112 
    +
    113 
    +    @property
    114 
    +    def method(self):
    115 
    +        return self._method
    116 
    +
    117 
    +    @property
    118 
    +    def algorithm(self):
    119 
    +        return self._algorithm
    120 
    +
    121 
    +    def intercept_service(self, continuation, handler_call_details):
    122 
    +        try:
    123 
    +            # Reject requests not carrying a token:
    124 
    +            bearer = dict(handler_call_details.invocation_metadata)['authorization']
    125 
    +
    126 
    +        except KeyError:
    127 
    +            self.__logger.error("Rejecting '{}' request: {}"
    128 
    +                                .format(handler_call_details.method.split('/')[-1],
    129 
    +                                        self.__auth_errors['missing-bearer']))
    130 
    +            return self.__terminators['missing-bearer']
    131 
    +
    132 
    +        # Reject requests with malformated bearer:
    133 
    +        if not bearer.startswith('Bearer '):
    134 
    +            self.__logger.error("Rejecting '{}' request: {}"
    135 
    +                                .format(handler_call_details.method.split('/')[-1],
    136 
    +                                        self.__auth_errors['invalid-bearer']))
    137 
    +            return self.__terminators['invalid-bearer']
    138 
    +
    139 
    +        try:
    140 
    +            # Hit the cache for already validated token:
    141 
    +            expiration_time = self.__bearer_cache[bearer]
    142 
    +
    143 
    +            # Accept request if cached token hasn't expired yet:
    144 
    +            if expiration_time < datetime.utcnow():
    145 
    +                return continuation(handler_call_details)  # Accepted
    146 
    +
    147 
    +        except KeyError:
    148 
    +            pass
    149 
    +
    150 
    +        assert self.__validator is not None
    151 
    +
    152 
    +        try:
    153 
    +            # Decode and validate the new token:
    154 
    +            expiration_time = self.__validator(bearer[7:])
    155 
    +
    156 
    +        except _InvalidTokenError as e:
    157 
    +            self.__logger.error("Rejecting '{}' request: {}; {}"
    158 
    +                                .format(handler_call_details.method.split('/')[-1],
    159 
    +                                        self.__auth_errors['invalid-token'], str(e)))
    160 
    +            return self.__terminators['invalid-token']
    161 
    +
    162 
    +        except _ExpiredTokenError as e:
    163 
    +            self.__logger.error("Rejecting '{}' request: {}; {}"
    164 
    +                                .format(handler_call_details.method.split('/')[-1],
    165 
    +                                        self.__auth_errors['expired-token'], str(e)))
    166 
    +            return self.__terminators['expired-token']
    167 
    +
    168 
    +        except _UnboundedTokenError as e:
    169 
    +            self.__logger.error("Rejecting '{}' request: {}; {}"
    170 
    +                                .format(handler_call_details.method.split('/')[-1],
    171 
    +                                        self.__auth_errors['unbounded-token'], str(e)))
    172 
    +            return self.__terminators['unbounded-token']
    173 
    +
    174 
    +        # Cache the validated token and store expiration time:
    175 
    +        self.__bearer_cache[bearer] = expiration_time
    176 
    +
    177 
    +        return continuation(handler_call_details)  # Accepted
    178 
    +
    179 
    +    # --- Private API: JWT ---
    180 
    +
    181 
    +    def _check_jwt_support(self, algorithm=AuthMetadataAlgorithm.UNSPECIFIED):
    182 
    +        """Ensures JWT and possible dependencies are available."""
    183 
    +        if not HAVE_JWT:
    184 
    +            raise InvalidArgumentError("JWT authorization method requires PyJWT")
    185 
    +
    186 
    +        try:
    187 
    +            if algorithm != AuthMetadataAlgorithm.UNSPECIFIED:
    188 
    +                jwt.register_algorithm(algorithm.value.upper(), None)
    189 
    +
    190 
    +        except TypeError:
    191 
    +            raise InvalidArgumentError("Algorithm not supported for JWT decoding: [{}]"
    192 
    +                                       .format(self._algorithm))
    193 
    +
    194 
    +        except ValueError:
    195 
    +            pass
    196 
    +
    197 
    +    def _validate_jwt_token(self, token):
    198 
    +        """Validates a JWT token and returns its expiry date."""
    199 
    +        if self._algorithm != AuthMetadataAlgorithm.UNSPECIFIED:
    200 
    +            algorithms = [self._algorithm.value.upper()]
    201 
    +        else:
    202 
    +            algorithms = None
    203 
    +
    204 
    +        try:
    205 
    +            payload = jwt.decode(token, self.__secret, algorithms=algorithms)
    206 
    +
    207 
    +        except jwt.exceptions.ExpiredSignatureError as e:
    208 
    +            raise _ExpiredTokenError(e)
    209 
    +
    210 
    +        except jwt.exceptions.InvalidTokenError as e:
    211 
    +            raise _InvalidTokenError(e)
    212 
    +
    213 
    +        if 'exp' not in payload or not isinstance(payload['exp'], int):
    214 
    +            raise _UnboundedTokenError("Missing 'exp' in payload")
    215 
    +
    216 
    +        return datetime.fromtimestamp(payload['exp'])
    217 
    +
    218 
    +
    219 
    +def _unary_unary_rpc_terminator(details):
    220 
    +
    221 
    +    def terminate(ignored_request, context):
    222 
    +        context.abort(grpc.StatusCode.UNAUTHENTICATED, details)
    223 
    +
    224 
    +    return grpc.unary_unary_rpc_method_handler(terminate)

  • buildgrid/server/instance.py
    ... ... @@ -29,6 +29,7 @@ import janus
    29 29 
     from buildgrid._enums import BotStatus, LogRecordLevel, MetricRecordDomain, MetricRecordType
    30 30 
     from buildgrid._protos.buildgrid.v2 import monitoring_pb2
    31 31 
     from buildgrid.server.actioncache.service import ActionCacheService
    32 
    +from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm, AuthMetadataServerInterceptor
    32 33 
     from buildgrid.server.bots.service import BotsService
    33 34 
     from buildgrid.server.capabilities.instance import CapabilitiesInstance
    34 35 
     from buildgrid.server.capabilities.service import CapabilitiesService
    ... ... @@ -47,11 +48,22 @@ class BuildGridServer:
    47 48 
         requisite services.
    48 49 
         """
    49 50
    50 
    -    def __init__(self, max_workers=None, monitor=False):
    51 
    +    def __init__(self, max_workers=None, monitor=False, auth_method=AuthMetadataMethod.NONE,
    52 
    +                 auth_secret=None, auth_algorithm=AuthMetadataAlgorithm.UNSPECIFIED):
    51 53 
             """Initializes a new :class:`BuildGridServer` instance.
    52 54
    53 55 
             Args:
    54 56 
                 max_workers (int, optional): A pool of max worker threads.
    57 
    +            monitor (bool, optional): Whether or not to globally activate server
    58 
    +                monitoring. Defaults to ``False``.
    59 
    +            auth_method (AuthMetadataMethod, optional): Authentication method to
    60 
    +                be used for request authorization. Defaults to ``NONE``.
    61 
    +            auth_secret (str, optional): The secret or key to be used for
    62 
    +                authorizing request using `auth_method`. Defaults to ``None``.
    63 
    +            auth_algorithm (AuthMetadataAlgorithm, optional): The crytographic
    64 
    +                algorithm to be uses in combination with `auth_secret` for
    65 
    +                authorizing request using `auth_method`. Defaults to
    66 
    +                ``UNSPECIFIED``.
    55 67 
             """
    56 68 
             self.__logger = logging.getLogger(__name__)
    57 69
    ... ... @@ -59,8 +71,17 @@ class BuildGridServer:
    59 71 
                 # Use max_workers default from Python 3.5+
    60 72 
                 max_workers = (os.cpu_count() or 1) * 5
    61 73
    74 
    +        self.__grpc_auth_interceptor = None
    75 
    +        if auth_method != AuthMetadataMethod.NONE:
    76 
    +            self.__grpc_auth_interceptor = AuthMetadataServerInterceptor(
    77 
    +                method=auth_method, secret=auth_secret, algorithm=auth_algorithm)
    62 78 
             self.__grpc_executor = futures.ThreadPoolExecutor(max_workers)
    63 
    -        self.__grpc_server = grpc.server(self.__grpc_executor)
    79 
    +
    80 
    +        if self.__grpc_auth_interceptor is not None:
    81 
    +            self.__grpc_server = grpc.server(
    82 
    +                self.__grpc_executor, interceptors=(self.__grpc_auth_interceptor,))
    83 
    +        else:
    84 
    +            self.__grpc_server = grpc.server(self.__grpc_executor)
    64 85
    65 86 
             self.__main_loop = asyncio.get_event_loop()
    66 87
    ... ... @@ -71,7 +92,10 @@ class BuildGridServer:
    71 92 
             self.__logging_formatter = logging.Formatter(fmt=LOG_RECORD_FORMAT)
    72 93 
             self.__print_log_records = True
    73 94
    95 
    +        self.__build_metadata_queues = None
    96 
    +
    74 97 
             self.__state_monitoring_task = None
    98 
    +        self.__build_monitoring_tasks = None
    75 99 
             self.__logging_task = None
    76 100
    77 101 
             # We always want a capabilities service
    ... ... @@ -95,6 +119,8 @@ class BuildGridServer:
    95 119 
                     self.__main_loop, endpoint_type=MonitoringOutputType.STDOUT,
    96 120 
                     serialisation_format=MonitoringOutputFormat.JSON)
    97 121
    122 
    +            self.__build_monitoring_tasks = []
    123 
    +
    98 124 
             # Setup the main logging handler:
    99 125 
             root_logger = logging.getLogger()
    100 126
    ... ... @@ -119,6 +145,18 @@ class BuildGridServer:
    119 145 
                     self._state_monitoring_worker(period=MONITORING_PERIOD),
    120 146 
                     loop=self.__main_loop)
    121 147
    148 
    +            self.__build_monitoring_tasks.clear()
    149 
    +            for instance_name, scheduler in self._schedulers.items():
    150 
    +                if not scheduler.is_instrumented:
    151 
    +                    continue
    152 
    +
    153 
    +                message_queue = janus.Queue(loop=self.__main_loop)
    154 
    +                scheduler.register_build_metadata_watcher(message_queue.sync_q)
    155 
    +
    156 
    +                self.__build_monitoring_tasks.append(asyncio.ensure_future(
    157 
    +                    self._build_monitoring_worker(instance_name, message_queue),
    158 
    +                    loop=self.__main_loop))
    159 
    +
    122 160 
             self.__logging_task = asyncio.ensure_future(
    123 161 
                 self._logging_worker(), loop=self.__main_loop)
    124 162
    ... ... @@ -132,6 +170,10 @@ class BuildGridServer:
    132 170 
                 if self.__state_monitoring_task is not None:
    133 171 
                     self.__state_monitoring_task.cancel()
    134 172
    173 
    +            for build_monitoring_task in self.__build_monitoring_tasks:
    174 
    +                build_monitoring_task.cancel()
    175 
    +            self.__build_monitoring_tasks.clear()
    176 
    +
    135 177 
                 self.__monitoring_bus.stop()
    136 178
    137 179 
             if self.__logging_task is not None:
    ... ... @@ -352,6 +394,60 @@ class BuildGridServer:
    352 394
    353 395 
             return log_record
    354 396
    397 
    +    async def _build_monitoring_worker(self, instance_name, message_queue):
    398 
    +        """Publishes builds metadata to the monitoring bus."""
    399 
    +        async def __build_monitoring_worker():
    400 
    +            metadata, context = await message_queue.async_q.get()
    401 
    +
    402 
    +            context.update({'instance-name': instance_name or ''})
    403 
    +
    404 
    +            # Emit build inputs fetching time record:
    405 
    +            fetch_start = metadata.input_fetch_start_timestamp.ToDatetime()
    406 
    +            fetch_completed = metadata.input_fetch_completed_timestamp.ToDatetime()
    407 
    +            input_fetch_time = fetch_completed - fetch_start
    408 
    +            timer_record = self._forge_timer_metric_record(
    409 
    +                MetricRecordDomain.BUILD, 'inputs-fetching-time', input_fetch_time,
    410 
    +                metadata=context)
    411 
    +
    412 
    +            await self.__monitoring_bus.send_record(timer_record)
    413 
    +
    414 
    +            # Emit build execution time record:
    415 
    +            execution_start = metadata.execution_start_timestamp.ToDatetime()
    416 
    +            execution_completed = metadata.execution_completed_timestamp.ToDatetime()
    417 
    +            execution_time = execution_completed - execution_start
    418 
    +            timer_record = self._forge_timer_metric_record(
    419 
    +                MetricRecordDomain.BUILD, 'execution-time', execution_time,
    420 
    +                metadata=context)
    421 
    +
    422 
    +            await self.__monitoring_bus.send_record(timer_record)
    423 
    +
    424 
    +            # Emit build outputs uploading time record:
    425 
    +            upload_start = metadata.output_upload_start_timestamp.ToDatetime()
    426 
    +            upload_completed = metadata.output_upload_completed_timestamp.ToDatetime()
    427 
    +            output_upload_time = upload_completed - upload_start
    428 
    +            timer_record = self._forge_timer_metric_record(
    429 
    +                MetricRecordDomain.BUILD, 'outputs-uploading-time', output_upload_time,
    430 
    +                metadata=context)
    431 
    +
    432 
    +            await self.__monitoring_bus.send_record(timer_record)
    433 
    +
    434 
    +            # Emit total build handling time record:
    435 
    +            queued = metadata.queued_timestamp.ToDatetime()
    436 
    +            worker_completed = metadata.worker_completed_timestamp.ToDatetime()
    437 
    +            total_handling_time = worker_completed - queued
    438 
    +            timer_record = self._forge_timer_metric_record(
    439 
    +                MetricRecordDomain.BUILD, 'total-handling-time', total_handling_time,
    440 
    +                metadata=context)
    441 
    +
    442 
    +            await self.__monitoring_bus.send_record(timer_record)
    443 
    +
    444 
    +        try:
    445 
    +            while True:
    446 
    +                await __build_monitoring_worker()
    447 
    +
    448 
    +        except asyncio.CancelledError:
    449 
    +            pass
    450 
    +
    355 451 
         async def _state_monitoring_worker(self, period=1.0):
    356 452 
             """Periodically publishes state metrics to the monitoring bus."""
    357 453 
             async def __state_monitoring_worker():
    ... ... @@ -463,7 +559,7 @@ class BuildGridServer:
    463 559 
             n_clients = self._execution_service.query_n_clients_for_instance(instance_name)
    464 560 
             gauge_record = self._forge_gauge_metric_record(
    465 561 
                 MetricRecordDomain.STATE, 'clients-count', n_clients,
    466 
    -            metadata={'instance-name': instance_name or 'void'})
    562 
    +            metadata={'instance-name': instance_name or ''})
    467 563
    468 564 
             return n_clients, gauge_record
    469 565
    ... ... @@ -480,7 +576,7 @@ class BuildGridServer:
    480 576 
             n_bots = self._bots_service.query_n_bots_for_instance(instance_name)
    481 577 
             gauge_record = self._forge_gauge_metric_record(
    482 578 
                 MetricRecordDomain.STATE, 'bots-count', n_bots,
    483 
    -            metadata={'instance-name': instance_name or 'void'})
    579 
    +            metadata={'instance-name': instance_name or ''})
    484 580
    485 581 
             return n_bots, gauge_record
    486 582
    ... ... @@ -498,6 +594,6 @@ class BuildGridServer:
    498 594 
             am_queue_time = self._schedulers[instance_name].query_am_queue_time()
    499 595 
             timer_record = self._forge_timer_metric_record(
    500 596 
                 MetricRecordDomain.STATE, 'average-queue-time', am_queue_time,
    501 
    -            metadata={'instance-name': instance_name or 'void'})
    597 
    +            metadata={'instance-name': instance_name or ''})
    502 598
    503 599 
             return am_queue_time, timer_record

  • buildgrid/server/job.py
    ... ... @@ -83,6 +83,13 @@ class Job:
    83 83 
             else:
    84 84 
                 return None
    85 85
    86 
    +    @property
    87 
    +    def holds_cached_action_result(self):
    88 
    +        if self.__execute_response is not None:
    89 
    +            return self.__execute_response.cached_result
    90 
    +        else:
    91 
    +            return False
    92 
    +
    86 93 
         @property
    87 94 
         def operation(self):
    88 95 
             return self._operation

  • buildgrid/server/scheduler.py
    ... ... @@ -34,6 +34,8 @@ class Scheduler:
    34 34 
         def __init__(self, action_cache=None, monitor=False):
    35 35 
             self.__logger = logging.getLogger(__name__)
    36 36
    37 
    +        self.__build_metadata_queues = None
    38 
    +
    37 39 
             self.__operations_by_stage = None
    38 40 
             self.__leases_by_state = None
    39 41 
             self.__queue_time_average = None
    ... ... @@ -46,6 +48,8 @@ class Scheduler:
    46 48 
             self._is_instrumented = monitor
    47 49
    48 50 
             if self._is_instrumented:
    51 
    +            self.__build_metadata_queues = []
    52 
    +
    49 53 
                 self.__operations_by_stage = {}
    50 54 
                 self.__leases_by_state = {}
    51 55 
                 self.__queue_time_average = 0, timedelta()
    ... ... @@ -228,6 +232,10 @@ class Scheduler:
    228 232 
         def is_instrumented(self):
    229 233 
             return self._is_instrumented
    230 234
    235 
    +    def register_build_metadata_watcher(self, message_queue):
    236 
    +        if self.__build_metadata_queues is not None:
    237 
    +            self.__build_metadata_queues.append(message_queue)
    238 
    +
    231 239 
         def query_n_jobs(self):
    232 240 
             return len(self.jobs)
    233 241
    ... ... @@ -319,3 +327,12 @@ class Scheduler:
    319 327 
                         average_time = average_time + ((queue_time - average_time) / average_order)
    320 328
    321 329 
                     self.__queue_time_average = average_order, average_time
    330 
    +
    331 
    +                if not job.holds_cached_action_result:
    332 
    +                    execution_metadata = job.action_result.execution_metadata
    333 
    +                    context_metadata = {'job-is': job.name}
    334 
    +
    335 
    +                    message = (execution_metadata, context_metadata,)
    336 
    +
    337 
    +                    for message_queue in self.__build_metadata_queues:
    338 
    +                        message_queue.put(message)

  • docs/source/installation.rst
    ... ... @@ -59,20 +59,20 @@ have to adjust your ``PATH``, in ``~/.bashrc``, with:
    59 59
    60 60 
     .. note::
    61 61
    62 
    -   The ``setup.py`` script defines two extra targets, ``docs`` and ``tests``,
    63 
    -   that declare required dependency for, respectively, generating documentation
    64 
    -   and running unit-tests. They can be use as helpers for setting up a
    65 
    -   development environment. To use them simply run:
    62 
    +   The ``setup.py`` script defines three extra targets, ``auth``, ``docs`` and
    63 
    +   ``tests``. They declare required dependency for, respectively, authentication
    64 
    +   and authorization management, generating documentation and running
    65 
    +   unit-tests. They can be use as helpers for setting up a development
    66 
    +   environment. To use them simply run:
    66 67
    67 68 
        .. code-block:: sh
    68 69
    69 
    -      pip3 install --user --editable ".[docs,tests]"
    70 
    -
    70 
    +      pip3 install --user --editable ".[auth,docs,tests]"
    71 71
    72 72
    73 73 
     .. install-docker:
    74 74
    75 
    -Installation through docker
    75 
    +Installation through Docker
    76 76 
     ---------------------------
    77 77
    78 78 
     How to build a Docker image that runs BuildGrid.

  • setup.py
    ... ... @@ -85,6 +85,11 @@ def get_cmdclass():
    85 85 
         }
    86 86 
         return cmdclass
    87 87
    88 
    +auth_require = [
    89 
    +    'cryptography >= 1.8.0',  # Required by pyjwt for RSA
    90 
    +    'pyjwt >= 1.5.0',
    91 
    +]
    92 
    +
    88 93 
     tests_require = [
    89 94 
         'coverage >= 4.5.0',
    90 95 
         'moto < 1.3.7',
    ... ... @@ -130,6 +135,7 @@ setup(
    130 135 
         setup_requires=['pytest-runner'],
    131 136 
         tests_require=tests_require,
    132 137 
         extras_require={
    138 
    +        'auth': auth_require,
    133 139 
             'docs': docs_require,
    134 140 
             'tests': tests_require,
    135 141 
         },

  • tests/auth/__init__.py

  • tests/auth/data/jwt-hs256-conflicting.secret
    1 
    +not-your-256-bit-secret

  • tests/auth/data/jwt-hs256-expired.token
    1 
    +eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMiwiZXhwIjoxMzU1MzE0MzMyfQ.J-YxkvVbeZFwZ2_mK1d91Wb5vm481LkuehfkUoNpLb8

  • tests/auth/data/jwt-hs256-matching.secret
    1 
    +your-256-bit-secret

  • tests/auth/data/jwt-hs256-unbounded.token
    1 
    +eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMn0.tEBYb8GM_4lmJCxrh6iMhdoxupgGTCaJR3atLxodZV8

  • tests/auth/data/jwt-hs256-valid.token
    1 
    +eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMiwiZXhwIjoyMzAxOTk5MTMyfQ.U__BJgksk65S0YuDZqU85tRWF3F03ITF9HXW1cd_Ci8

  • tests/auth/data/jwt-rs256-conflicting.pub.key
    1 
    +-----BEGIN PUBLIC KEY-----
    2 
    +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDNGWI+5SgYIiGcS5UNQzZOs5As
    3 
    +QJRsKR7bBiMjJWnqrNExn0pYrecQsBy6zgwmN6MqEPFV5A1GOEeP3FAqlwD5y+rL
    4 
    +iO2tqq0naiFiJb27qsHzgjakw7pVqgJuGuLVIWWE1RlDnhfN+auNGWyl2YjF6N2+
    5 
    +Lsl0bBX8Q8zaOxbU3wIDAQAB
    6 
    +-----END PUBLIC KEY-----

  • tests/auth/data/jwt-rs256-expired.token
    1 
    +eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMiwiZXhwIjoxMzU1MzE0MzMyfQ.L9Wg6YDEV57_a_j_J2s6ug5eEGS80SvlwztdVFzU1ajqlfYmbF5oYq6IEMaiZq95aKkJ71xnwpGfpcuufH-xONiNoZhTg7r-lb99yvPZ8VEHwOIyt1EUEziffim9XRiuwM570fg7_HUC-ZhNJG536k1IM-6rPRnv-1Tu-MxLgvQ

  • tests/auth/data/jwt-rs256-matching.priv.key
    1 
    +-----BEGIN RSA PRIVATE KEY-----
    2 
    +MIICWwIBAAKBgQDdlatRjRjogo3WojgGHFHYLugdUWAY9iR3fy4arWNA1KoS8kVw
    3 
    +33cJibXr8bvwUAUparCwlvdbH6dvEOfou0/gCFQsHUfQrSDv+MuSUMAe8jzKE4qW
    4 
    ++jK+xQU9a03GUnKHkkle+Q0pX/g6jXZ7r1/xAK5Do2kQ+X5xK9cipRgEKwIDAQAB
    5 
    +AoGAD+onAtVye4ic7VR7V50DF9bOnwRwNXrARcDhq9LWNRrRGElESYYTQ6EbatXS
    6 
    +3MCyjjX2eMhu/aF5YhXBwkppwxg+EOmXeh+MzL7Zh284OuPbkglAaGhV9bb6/5Cp
    7 
    +uGb1esyPbYW+Ty2PC0GSZfIXkXs76jXAu9TOBvD0ybc2YlkCQQDywg2R/7t3Q2OE
    8 
    +2+yo382CLJdrlSLVROWKwb4tb2PjhY4XAwV8d1vy0RenxTB+K5Mu57uVSTHtrMK0
    9 
    +GAtFr833AkEA6avx20OHo61Yela/4k5kQDtjEf1N0LfI+BcWZtxsS3jDM3i1Hp0K
    10 
    +Su5rsCPb8acJo5RO26gGVrfAsDcIXKC+bQJAZZ2XIpsitLyPpuiMOvBbzPavd4gY
    11 
    +6Z8KWrfYzJoI/Q9FuBo6rKwl4BFoToD7WIUS+hpkagwWiz+6zLoX1dbOZwJACmH5
    12 
    +fSSjAkLRi54PKJ8TFUeOP15h9sQzydI8zJU+upvDEKZsZc/UhT/SySDOxQ4G/523
    13 
    +Y0sz/OZtSWcol/UMgQJALesy++GdvoIDLfJX5GBQpuFgFenRiRDabxrE9MNUZ2aP
    14 
    +FaFp+DyAe+b4nDwuJaW2LURbr8AEZga7oQj0uYxcYw==
    15 
    +-----END RSA PRIVATE KEY-----
    \ No newline at end of file

  • tests/auth/data/jwt-rs256-matching.pub.key
    1 
    +-----BEGIN PUBLIC KEY-----
    2 
    +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDdlatRjRjogo3WojgGHFHYLugd
    3 
    +UWAY9iR3fy4arWNA1KoS8kVw33cJibXr8bvwUAUparCwlvdbH6dvEOfou0/gCFQs
    4 
    +HUfQrSDv+MuSUMAe8jzKE4qW+jK+xQU9a03GUnKHkkle+Q0pX/g6jXZ7r1/xAK5D
    5 
    +o2kQ+X5xK9cipRgEKwIDAQAB
    6 
    +-----END PUBLIC KEY-----

  • tests/auth/data/jwt-rs256-unbounded.token
    1 
    +eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMn0.1_yoF5Vg1fXs2SmWhYm7LbLMGbZVgBjxZkzRlw87blSG6lRgEi_R-WXKcS4n_pGynkcahJ_AqLseyHduXZveI1nVQFATXVNQQPcvmkM6pYSHPm155iqZFdYAWWVKB9ND1F3oDrXLBzqF2a4HtLNXYTu5gDStdUPkrH_FiatX79g

  • tests/auth/data/jwt-rs256-valid.token
    1 
    +eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJCdWlsZEdyaWQgVGVzdCIsImlhdCI6MTM1NTMxNDMzMiwiZXhwIjoyMzAxOTk5MTMyfQ.RCbQNqPaF0mfFsUGwdb47Ga3DITAL7OjYlcWkJ2xWL61Fo9zURx_mSIVDYTgEY3nFW1cmf2r6Y2Z0rEUOY-qBl8B9Ww9jijGz7LMR4w_j8f967MjTxkyOCWZSUWazObg5jxNjLtfxPD28UbbrS_2R8BLgMQFf3ymSDXTX5lAdBY

  • tests/auth/test_client.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +# pylint: disable=redefined-outer-name
    16 
    +
    17 
    +
    18 
    +import os
    19 
    +
    20 
    +import grpc
    21 
    +import pytest
    22 
    +
    23 
    +from buildgrid.client.authentication import setup_channel
    24 
    +from buildgrid._protos.google.bytestream import bytestream_pb2, bytestream_pb2_grpc
    25 
    +from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm
    26 
    +
    27 
    +from ..utils.dummy import serve_dummy
    28 
    +from ..utils.utils import run_in_subprocess
    29 
    +
    30 
    +
    31 
    +try:
    32 
    +    import jwt  # pylint: disable=unused-import
    33 
    +except ImportError:
    34 
    +    HAVE_JWT = False
    35 
    +else:
    36 
    +    HAVE_JWT = True
    37 
    +
    38 
    +
    39 
    +DATA_DIR = os.path.join(
    40 
    +    os.path.dirname(os.path.realpath(__file__)), 'data')
    41 
    +
    42 
    +METHOD = AuthMetadataMethod.JWT
    43 
    +TOKEN = os.path.join(DATA_DIR, 'jwt-hs256-valid.token')
    44 
    +SECRET = 'your-256-bit-secret'
    45 
    +ALGORITHM = AuthMetadataAlgorithm.JWT_HS256
    46 
    +
    47 
    +
    48 
    +@pytest.mark.skipif(not HAVE_JWT, reason="No pyjwt")
    49 
    +def test_channel_token_authorization():
    50 
    +    # Actual test function, to be run in a subprocess:
    51 
    +    def __test_channel_token_authorization(queue, remote, token):
    52 
    +        channel, _ = setup_channel(remote, auth_token=token)
    53 
    +        stub = bytestream_pb2_grpc.ByteStreamStub(channel)
    54 
    +
    55 
    +        request = bytestream_pb2.QueryWriteStatusRequest()
    56 
    +        status_code = grpc.StatusCode.OK
    57 
    +
    58 
    +        try:
    59 
    +            next(stub.Read(request))
    60 
    +
    61 
    +        except grpc.RpcError as e:
    62 
    +            status_code = e.code()
    63 
    +
    64 
    +        queue.put(status_code)
    65 
    +
    66 
    +    with serve_dummy(auth_method=METHOD, auth_secret=SECRET,
    67 
    +                     auth_algorithm=ALGORITHM) as server:
    68 
    +        status = run_in_subprocess(__test_channel_token_authorization,
    69 
    +                                   server.remote, TOKEN)
    70 
    +
    71 
    +        assert status != grpc.StatusCode.UNAUTHENTICATED

  • tests/auth/test_interceptor.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +# pylint: disable=redefined-outer-name
    16 
    +
    17 
    +
    18 
    +from collections import namedtuple
    19 
    +from unittest import mock
    20 
    +import os
    21 
    +
    22 
    +import grpc
    23 
    +from grpc._server import _Context
    24 
    +import pytest
    25 
    +
    26 
    +from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm
    27 
    +from buildgrid.server._authentication import AuthMetadataServerInterceptor
    28 
    +
    29 
    +from ..utils.utils import read_file
    30 
    +
    31 
    +
    32 
    +try:
    33 
    +    import jwt  # pylint: disable=unused-import
    34 
    +except ImportError:
    35 
    +    HAVE_JWT = False
    36 
    +else:
    37 
    +    HAVE_JWT = True
    38 
    +
    39 
    +
    40 
    +DATA_DIR = os.path.join(
    41 
    +    os.path.dirname(os.path.realpath(__file__)), 'data')
    42 
    +
    43 
    +TOKENS = [None, 'not-a-token']
    44 
    +SECRETS = [None, None]
    45 
    +ALGORITHMS = [
    46 
    +    AuthMetadataAlgorithm.UNSPECIFIED,
    47 
    +    AuthMetadataAlgorithm.UNSPECIFIED]
    48 
    +VALIDITIES = [False, False]
    49 
    +# Generic test data: token, secret, algorithm, validity:
    50 
    +DATA = zip(TOKENS, SECRETS, ALGORITHMS, VALIDITIES)
    51 
    +
    52 
    +JWT_TOKENS = [
    53 
    +    'jwt-hs256-expired.token',
    54 
    +    'jwt-hs256-unbounded.token',
    55 
    +    'jwt-hs256-valid.token',
    56 
    +    'jwt-hs256-valid.token',
    57 
    +
    58 
    +    'jwt-rs256-expired.token',
    59 
    +    'jwt-rs256-unbounded.token',
    60 
    +    'jwt-rs256-valid.token',
    61 
    +    'jwt-rs256-valid.token']
    62 
    +JWT_SECRETS = [
    63 
    +    'jwt-hs256-matching.secret',
    64 
    +    'jwt-hs256-matching.secret',
    65 
    +    'jwt-hs256-matching.secret',
    66 
    +    'jwt-hs256-conflicting.secret',
    67 
    +
    68 
    +    'jwt-rs256-matching.pub.key',
    69 
    +    'jwt-rs256-matching.pub.key',
    70 
    +    'jwt-rs256-matching.pub.key',
    71 
    +    'jwt-rs256-conflicting.pub.key']
    72 
    +JWT_ALGORITHMS = [
    73 
    +    AuthMetadataAlgorithm.JWT_HS256,
    74 
    +    AuthMetadataAlgorithm.JWT_HS256,
    75 
    +    AuthMetadataAlgorithm.UNSPECIFIED,
    76 
    +    AuthMetadataAlgorithm.JWT_HS256,
    77 
    +
    78 
    +    AuthMetadataAlgorithm.JWT_RS256,
    79 
    +    AuthMetadataAlgorithm.JWT_RS256,
    80 
    +    AuthMetadataAlgorithm.UNSPECIFIED,
    81 
    +    AuthMetadataAlgorithm.JWT_RS256]
    82 
    +JWT_VALIDITIES = [
    83 
    +    False, False, True, False,
    84 
    +    False, False, True, False]
    85 
    +# JWT test data: token, secret, algorithm, validity:
    86 
    +JWT_DATA = zip(JWT_TOKENS, JWT_SECRETS, JWT_ALGORITHMS, JWT_VALIDITIES)
    87 
    +
    88 
    +
    89 
    +_MockHandlerCallDetails = namedtuple(
    90 
    +    '_MockHandlerCallDetails', ('method', 'invocation_metadata',))
    91 
    +_MockMetadatum = namedtuple(
    92 
    +    '_MockMetadatum', ('key', 'value',))
    93 
    +
    94 
    +
    95 
    +def _mock_call_details(token, method='TestMethod'):
    96 
    +    invocation_metadata = [
    97 
    +        _MockMetadatum(
    98 
    +            key='user-agent',
    99 
    +            value='grpc-c/6.0.0 (manylinux; chttp2; gao)')]
    100 
    +
    101 
    +    if token and token.count('.') == 2:
    102 
    +        invocation_metadata.append(_MockMetadatum(
    103 
    +            key='authorization', value='Bearer {}'.format(token)))
    104 
    +
    105 
    +    elif token:
    106 
    +        invocation_metadata.append(_MockMetadatum(
    107 
    +            key='authorization', value=token))
    108 
    +
    109 
    +    return _MockHandlerCallDetails(
    110 
    +        method=method, invocation_metadata=invocation_metadata)
    111 
    +
    112 
    +
    113 
    +def _unary_unary_rpc_terminator(details):
    114 
    +
    115 
    +    def terminate(ignored_request, context):
    116 
    +        context.set_code(grpc.StatusCode.OK)
    117 
    +
    118 
    +    return grpc.unary_unary_rpc_method_handler(terminate)
    119 
    +
    120 
    +
    121 
    +@pytest.mark.parametrize('token,secret,algorithm,validity', DATA)
    122 
    +def test_authorization(token, secret, algorithm, validity):
    123 
    +    interceptor = AuthMetadataServerInterceptor(
    124 
    +        method=AuthMetadataMethod.NONE, secret=secret, algorithm=algorithm)
    125 
    +
    126 
    +    call_details = _mock_call_details(token)
    127 
    +    context = mock.create_autospec(_Context, spec_set=True)
    128 
    +
    129 
    +    try:
    130 
    +        handler = interceptor.intercept_service(None, call_details)
    131 
    +
    132 
    +    except AssertionError:
    133 
    +        context.set_code(grpc.StatusCode.OK)
    134 
    +
    135 
    +    else:
    136 
    +        handler.unary_unary(None, context)
    137 
    +
    138 
    +    if validity:
    139 
    +        context.set_code.assert_called_once_with(grpc.StatusCode.OK)
    140 
    +        context.abort.assert_not_called()
    141 
    +
    142 
    +    else:
    143 
    +        context.abort.assert_called_once_with(grpc.StatusCode.UNAUTHENTICATED, mock.ANY)
    144 
    +        context.set_code.assert_not_called()
    145 
    +
    146 
    +
    147 
    +@pytest.mark.skipif(not HAVE_JWT, reason="No pyjwt")
    148 
    +@pytest.mark.parametrize('token,secret,algorithm,validity', JWT_DATA)
    149 
    +def test_jwt_authorization(token, secret, algorithm, validity):
    150 
    +    token = read_file(os.path.join(DATA_DIR, token), text_mode=True).strip()
    151 
    +    secret = read_file(os.path.join(DATA_DIR, secret), text_mode=True).strip()
    152 
    +
    153 
    +    interceptor = AuthMetadataServerInterceptor(
    154 
    +        method=AuthMetadataMethod.JWT, secret=secret, algorithm=algorithm)
    155 
    +
    156 
    +    continuator = _unary_unary_rpc_terminator
    157 
    +    call_details = _mock_call_details(token)
    158 
    +    context = mock.create_autospec(_Context, spec_set=True)
    159 
    +
    160 
    +    handler = interceptor.intercept_service(continuator, call_details)
    161 
    +    handler.unary_unary(None, context)
    162 
    +
    163 
    +    if validity:
    164 
    +        context.set_code.assert_called_once_with(grpc.StatusCode.OK)
    165 
    +        context.abort.assert_not_called()
    166 
    +
    167 
    +    else:
    168 
    +        context.abort.assert_called_once_with(grpc.StatusCode.UNAUTHENTICATED, mock.ANY)
    169 
    +        context.set_code.assert_not_called()

  • tests/utils/dummy.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +from concurrent import futures
    17 
    +from contextlib import contextmanager
    18 
    +import multiprocessing
    19 
    +import os
    20 
    +import signal
    21 
    +
    22 
    +import grpc
    23 
    +import pytest_cov
    24 
    +
    25 
    +from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm
    26 
    +from buildgrid.server._authentication import AuthMetadataServerInterceptor
    27 
    +
    28 
    +
    29 
    +@contextmanager
    30 
    +def serve_dummy(auth_method=AuthMetadataMethod.NONE,
    31 
    +                auth_secret=None, auth_algorithm=AuthMetadataAlgorithm.UNSPECIFIED):
    32 
    +    server = Server(
    33 
    +        auth_method=auth_method, auth_secret=auth_secret, auth_algorithm=auth_algorithm)
    34 
    +    try:
    35 
    +        yield server
    36 
    +    finally:
    37 
    +        server.quit()
    38 
    +
    39 
    +
    40 
    +class Server:
    41 
    +
    42 
    +    def __init__(self, auth_method=AuthMetadataMethod.NONE,
    43 
    +                 auth_secret=None, auth_algorithm=AuthMetadataAlgorithm.UNSPECIFIED):
    44 
    +
    45 
    +        self.__queue = multiprocessing.Queue()
    46 
    +        self.__auth_interceptor = None
    47 
    +        if auth_method != AuthMetadataMethod.NONE:
    48 
    +            self.__auth_interceptor = AuthMetadataServerInterceptor(
    49 
    +                method=auth_method, secret=auth_secret, algorithm=auth_algorithm)
    50 
    +        self.__process = multiprocessing.Process(
    51 
    +            target=Server.serve, args=(self.__queue, self.__auth_interceptor))
    52 
    +        self.__process.start()
    53 
    +
    54 
    +        self.port = self.__queue.get()
    55 
    +        self.remote = 'http://localhost:{}'.format(self.port)
    56 
    +
    57 
    +    @classmethod
    58 
    +    def serve(cls, queue, auth_interceptor):
    59 
    +        pytest_cov.embed.cleanup_on_sigterm()
    60 
    +
    61 
    +        # Use max_workers default from Python 3.5+
    62 
    +        max_workers = (os.cpu_count() or 1) * 5
    63 
    +        executor = futures.ThreadPoolExecutor(max_workers)
    64 
    +        if auth_interceptor is not None:
    65 
    +            server = grpc.server(executor, interceptors=(auth_interceptor,))
    66 
    +        else:
    67 
    +            server = grpc.server(executor)
    68 
    +        port = server.add_insecure_port('localhost:0')
    69 
    +
    70 
    +        queue.put(port)
    71 
    +
    72 
    +        server.start()
    73 
    +
    74 
    +        signal.pause()
    75 
    +
    76 
    +    def quit(self):
    77 
    +        if self.__process:
    78 
    +            self.__process.terminate()
    79 
    +            self.__process.join()

  • tests/utils/utils.py
    ... ... @@ -34,6 +34,11 @@ def kill_process_tree(pid):
    34 34 
         kill_proc(proc)
    35 35
    36 36
    37 
    +def read_file(file_path, text_mode=False):
    38 
    +    with open(file_path, 'r' if text_mode else 'rb') as in_file:
    39 
    +        return in_file.read()
    40 
    +
    41 
    +
    37 42 
     def run_in_subprocess(function, *arguments, timeout=1):
    38 43 
         queue = multiprocessing.Queue()
    39 44 
         # Use subprocess to avoid creation of gRPC threads in main process

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]