GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][mablanch/156-bgd-action-cache] 6 commits: cli.py: Better handle errors during command invocations

Title: GitLab

Martin Blanchard pushed to branch mablanch/156-bgd-action-cache at BuildGrid / buildgrid

Commits:

7 changed files:

Changes:

  • buildgrid/_app/cli.py
    ... ... @@ -21,6 +21,7 @@ Any files in the commands/ folder with the name cmd_*.py
    21 21 
     will be attempted to be imported.
    22 22 
     """
    23 23
    24 
    +import importlib
    24 25 
     import logging
    25 26 
     import os
    26 27 
     import sys
    ... ... @@ -123,21 +124,30 @@ cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__),
    123 124 
                                               'commands'))
    124 125
    125 126
    126 
    -class BuildGridCLI(click.MultiCommand):
    127 
    +class App(click.MultiCommand):
    127 128
    128 129 
         def list_commands(self, context):
    130 
    +        """Lists available command names."""
    129 131 
             commands = []
    130 132 
             for filename in os.listdir(cmd_folder):
    131 
    -            if filename.endswith('.py') and \
    132 
    -               filename.startswith('cmd_'):
    133 
    +            if filename.endswith('.py') and filename.startswith('cmd_'):
    133 134 
                     commands.append(filename[4:-3])
    134 135 
             commands.sort()
    136 
    +
    135 137 
             return commands
    136 138
    137 
    -    def get_command(self, context, name):
    138 
    -        mod = __import__(name='buildgrid._app.commands.cmd_{}'.format(name),
    139 
    -                         fromlist=['cli'])
    140 
    -        return mod.cli
    139 
    +    def get_command(self, context, command_name):
    140 
    +        """Looks-up and loads a particular command by name."""
    141 
    +        command_name = command_name.replace('-', '')
    142 
    +        try:
    143 
    +            module = importlib.import_module(
    144 
    +                'buildgrid._app.commands.cmd_{}'.format(command_name))
    145 
    +
    146 
    +        except ImportError:
    147 
    +            click.echo("Error: No such command: [{}].".format(command_name), err=True)
    148 
    +            sys.exit(-1)
    149 
    +
    150 
    +        return module.cli
    141 151
    142 152
    143 153 
     class DebugFilter(logging.Filter):
    ... ... @@ -192,10 +202,10 @@ def setup_logging(verbosity=0, debug_mode=False):
    192 202 
             root_logger.setLevel(logging.DEBUG)
    193 203
    194 204
    195 
    -@click.command(cls=BuildGridCLI, context_settings=CONTEXT_SETTINGS)
    205 
    +@click.command(cls=App, context_settings=CONTEXT_SETTINGS)
    196 206 
     @pass_context
    197 207 
     def cli(context):
    198 
    -    """BuildGrid App"""
    208 
    +    """BuildGrid's client and server CLI front-end."""
    199 209 
         root_logger = logging.getLogger()
    200 210
    201 211 
         # Clean-up root logger for any pre-configuration:

  • buildgrid/_app/commands/cmd_actioncache.py
    1 
    +# Copyright (C) 2019 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +import os
    17 
    +import sys
    18 
    +from textwrap import indent
    19 
    +
    20 
    +import click
    21 
    +from google.protobuf import json_format
    22 
    +
    23 
    +from buildgrid.client.actioncache import query
    24 
    +from buildgrid.client.authentication import setup_channel
    25 
    +from buildgrid.client.cas import download
    26 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    27 
    +from buildgrid.utils import create_digest, parse_digest
    28 
    +
    29 
    +from ..cli import pass_context
    30 
    +
    31 
    +
    32 
    +@click.group(name='action-cache', short_help="Query and update the action cache service.")
    33 
    +@click.option('--remote', type=click.STRING, default='http://localhost:50051', show_default=True,
    34 
    +              help="Remote execution server's URL (port defaults to 50051 if no specified).")
    35 
    +@click.option('--auth-token', type=click.Path(exists=True, dir_okay=False), default=None,
    36 
    +              help="Authorization token for the remote.")
    37 
    +@click.option('--client-key', type=click.Path(exists=True, dir_okay=False), default=None,
    38 
    +              help="Private client key for TLS (PEM-encoded).")
    39 
    +@click.option('--client-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    40 
    +              help="Public client certificate for TLS (PEM-encoded).")
    41 
    +@click.option('--server-cert', type=click.Path(exists=True, dir_okay=False), default=None,
    42 
    +              help="Public server certificate for TLS (PEM-encoded)")
    43 
    +@click.option('--instance-name', type=click.STRING, default=None, show_default=True,
    44 
    +              help="Targeted farm instance name.")
    45 
    +@pass_context
    46 
    +def cli(context, remote, instance_name, auth_token, client_key, client_cert, server_cert):
    47 
    +    """Entry-point for the ``bgd action-cache`` CLI command group."""
    48 
    +    try:
    49 
    +        context.channel, _ = setup_channel(remote, auth_token=auth_token,
    50 
    +                                           client_key=client_key, client_cert=client_cert,
    51 
    +                                           server_cert=server_cert)
    52 
    +
    53 
    +    except InvalidArgumentError as e:
    54 
    +        click.echo("Error: {}.".format(e), err=True)
    55 
    +        sys.exit(-1)
    56 
    +
    57 
    +    context.instance_name = instance_name
    58 
    +
    59 
    +
    60 
    +@cli.command('get', short_help="Retrieves a cached action-result.")
    61 
    +@click.argument('action-digest-string', nargs=1, type=click.STRING, required=True)
    62 
    +@click.option('--json', is_flag=True, show_default=True,
    63 
    +              help="Print action result in JSON format.")
    64 
    +@pass_context
    65 
    +def get(context, action_digest_string, json):
    66 
    +    """Entry-point of the ``bgd action-cache get`` CLI command.
    67 
    +
    68 
    +    Note:
    69 
    +        Digest strings are expected to be like: ``{hash}/{size_bytes}``.
    70 
    +    """
    71 
    +    action_digest = parse_digest(action_digest_string)
    72 
    +    if action_digest is None:
    73 
    +        click.echo("Error: Invalid digest string '{}'.".format(action_digest_string), err=True)
    74 
    +        sys.exit(-1)
    75 
    +
    76 
    +    # Simply hit the action cache with the given action digest:
    77 
    +    with query(context.channel, instance=context.instance_name) as action_cache:
    78 
    +        action_result = action_cache.get(action_digest)
    79 
    +
    80 
    +    if action_result is not None:
    81 
    +        if not json:
    82 
    +            action_result_digest = create_digest(action_result.SerializeToString())
    83 
    +
    84 
    +            click.echo("Hit: {}/{}: Result cached with digest=[{}/{}]"
    85 
    +                       .format(action_digest.hash[:8], action_digest.size_bytes,
    86 
    +                               action_result_digest.hash, action_result_digest.size_bytes))
    87 
    +
    88 
    +            # TODO: Print ActionResult details?
    89 
    +
    90 
    +        else:
    91 
    +            click.echo(json_format.MessageToJson(action_result))
    92 
    +
    93 
    +    else:
    94 
    +        click.echo("Miss: {}/{}: No associated result found in cache..."
    95 
    +                   .format(action_digest.hash[:8], action_digest.size_bytes))
    96 
    +
    97 
    +
    98 
    +@cli.command('update', short_help="Maps an action to a given action-result.")
    99 
    +@click.argument('action-digest-string', nargs=1, type=click.STRING, required=True)
    100 
    +@click.argument('action-result-digest-string', nargs=1, type=click.STRING, required=True)
    101 
    +@pass_context
    102 
    +def update(context, action_digest_string, action_result_digest_string):
    103 
    +    """Entry-point of the ``bgd action-cache update`` CLI command.
    104 
    +
    105 
    +    Note:
    106 
    +        Digest strings are expected to be like: ``{hash}/{size_bytes}``.
    107 
    +    """
    108 
    +    action_digest = parse_digest(action_digest_string)
    109 
    +    if action_digest is None:
    110 
    +        click.echo("Error: Invalid digest string '{}'.".format(action_digest_string), err=True)
    111 
    +        sys.exit(-1)
    112 
    +
    113 
    +    action_result_digest = parse_digest(action_result_digest_string)
    114 
    +    if action_result_digest is None:
    115 
    +        click.echo("Error: Invalid digest string '{}'.".format(action_result_digest_string), err=True)
    116 
    +        sys.exit(-1)
    117 
    +
    118 
    +    # We have to download the ActionResult message fom CAS first...
    119 
    +    with download(context.channel, instance=context.instance_name) as downloader:
    120 
    +        action_result = downloader.get_message(action_result_digest,
    121 
    +                                               remote_execution_pb2.ActionResult())
    122 
    +
    123 
    +        # And only then we can update the action cache for the given digest:
    124 
    +        with query(context.channel, instance=context.instance_name) as action_cache:
    125 
    +            action_result = action_cache.update(action_digest, action_result)
    126 
    +
    127 
    +            if action_result is None:
    128 
    +                click.echo("Error: Failed updating cache result for action="">
    129 
    +                           .format(action_digest.hash, action_digest.size_bytes), err=True)
    130 
    +                sys.exit(-1)

  • buildgrid/_app/commands/cmd_cas.py
    ... ... @@ -29,7 +29,7 @@ from buildgrid.client.authentication import setup_channel
    29 29 
     from buildgrid.client.cas import download, upload
    30 30 
     from buildgrid._exceptions import InvalidArgumentError
    31 31 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    32 
    -from buildgrid.utils import create_digest, merkle_tree_maker, read_file
    32 
    +from buildgrid.utils import create_digest, parse_digest, merkle_tree_maker, read_file
    33 33
    34 34 
     from ..cli import pass_context
    35 35
    ... ... @@ -131,16 +131,6 @@ def upload_directory(context, directory_path, verify):
    131 131 
                 click.echo("Error: Failed pushing path=[{}]".format(node_path), err=True)
    132 132
    133 133
    134 
    -def _create_digest(digest_string):
    135 
    -    digest_hash, digest_size = digest_string.split('/')
    136 
    -
    137 
    -    digest = remote_execution_pb2.Digest()
    138 
    -    digest.hash = digest_hash
    139 
    -    digest.size_bytes = int(digest_size)
    140 
    -
    141 
    -    return digest
    142 
    -
    143 
    -
    144 134 
     @cli.command('download-file', short_help="Download one or more files from the CAS server. "
    145 135 
                                              "(Specified as a space-separated list of DIGEST FILE_PATH)")
    146 136 
     @click.argument('digest-path-list', nargs=-1, type=str, required=True)  # 'digest path' pairs
    ... ... @@ -158,7 +148,7 @@ def download_file(context, digest_path_list, verify):
    158 148 
                                "path=[{}] already exists.".format(file_path), err=True)
    159 149 
                     continue
    160 150
    161 
    -            digest = _create_digest(digest_string)
    151 
    +            digest = parse_digest(digest_string)
    162 152
    163 153 
                 downloader.download_file(digest, file_path)
    164 154 
                 downloaded_files[file_path] = digest
    ... ... @@ -191,7 +181,7 @@ def download_directory(context, digest_string, directory_path, verify):
    191 181 
                            "path=[{}] already exists.".format(directory_path), err=True)
    192 182 
                 return
    193 183
    194 
    -    digest = _create_digest(digest_string)
    184 
    +    digest = parse_digest(digest_string)
    195 185 
         with download(context.channel, instance=context.instance_name) as downloader:
    196 186 
             downloader.download_directory(digest, directory_path)
    197 187

  • buildgrid/client/actioncache.py
    1 
    +# Copyright (C) 2019 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +from contextlib import contextmanager
    17 
    +
    18 
    +import grpc
    19 
    +
    20 
    +from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    21 
    +
    22 
    +
    23 
    +@contextmanager
    24 
    +def query(channel, instance=None):
    25 
    +    """Context manager generator for the :class:`ActionCacheClient` class."""
    26 
    +    client = ActionCacheClient(channel, instance=instance)
    27 
    +    try:
    28 
    +        yield client
    29 
    +    finally:
    30 
    +        client.close()
    31 
    +
    32 
    +
    33 
    +class ActionCacheClient:
    34 
    +    """Remote ActionCache service client helper.
    35 
    +
    36 
    +    The :class:`ActionCacheClient` class comes with a generator factory function
    37 
    +    that can be used together with the `with` statement for context management::
    38 
    +
    39 
    +        from buildgrid.client.actioncache import query
    40 
    +
    41 
    +        with query(channel, instance='build') as action_cache:
    42 
    +            digest, action_result = action_cache.get(action_digest)
    43 
    +    """
    44 
    +
    45 
    +    def __init__(self, channel, instance=None):
    46 
    +        """Initializes a new :class:`ActionCacheClient` instance.
    47 
    +
    48 
    +        Args:
    49 
    +            channel (grpc.Channel): a gRPC channel to the ActionCache endpoint.
    50 
    +            instance (str, optional): the targeted instance's name.
    51 
    +        """
    52 
    +        self.channel = channel
    53 
    +
    54 
    +        self.instance_name = instance
    55 
    +
    56 
    +        self.__actioncache_stub = remote_execution_pb2_grpc.ActionCacheStub(self.channel)
    57 
    +
    58 
    +    # --- Public API ---
    59 
    +
    60 
    +    def get(self, action_digest):
    61 
    +        """Retrieves the cached :obj:`ActionResult` for a given :obj:`Action`.
    62 
    +
    63 
    +        Args:
    64 
    +            action_digest (:obj:`Digest`): the action's digest to query.
    65 
    +
    66 
    +        Returns:
    67 
    +            :obj:`ActionResult`: the cached result or None if not found.
    68 
    +
    69 
    +        Raises:
    70 
    +            grpc.RpcError: on any network or remote service error.
    71 
    +        """
    72 
    +        request = remote_execution_pb2.GetActionResultRequest()
    73 
    +        if self.instance_name:
    74 
    +            request.instance_name = self.instance_name
    75 
    +        request.action_digest.CopyFrom(action_digest)
    76 
    +
    77 
    +        try:
    78 
    +            return self.__actioncache_stub.GetActionResult(request)
    79 
    +
    80 
    +        except grpc.RpcError as e:
    81 
    +            status_code = e.code()
    82 
    +            if status_code != grpc.StatusCode.NOT_FOUND:
    83 
    +                raise
    84 
    +
    85 
    +        return None
    86 
    +
    87 
    +    def update(self, action_digest, action_result):
    88 
    +        """Maps in cache an :obj:`Action` to an :obj:`ActionResult`.
    89 
    +
    90 
    +        Args:
    91 
    +            action_digest (:obj:`Digest`): the action's digest to update.
    92 
    +            action_result (:obj:`ActionResult`): the action's result.
    93 
    +
    94 
    +        Returns:
    95 
    +            :obj:`ActionResult`: the cached result or None on failure.
    96 
    +
    97 
    +        Raises:
    98 
    +            grpc.RpcError: on any network or remote service error.
    99 
    +        """
    100 
    +        request = remote_execution_pb2.UpdateActionResultRequest()
    101 
    +        if self.instance_name:
    102 
    +            request.instance_name = self.instance_name
    103 
    +        request.action_digest.CopyFrom(action_digest)
    104 
    +        request.action_result.CopyFrom(action_result)
    105 
    +
    106 
    +        try:
    107 
    +            return self.__actioncache_stub.UpdateActionResult(request)
    108 
    +
    109 
    +        except grpc.RpcError as e:
    110 
    +            status_code = e.code()
    111 
    +            if status_code != grpc.StatusCode.NOT_FOUND:
    112 
    +                raise
    113 
    +
    114 
    +        return None
    115 
    +
    116 
    +    def close(self):
    117 
    +        """Closes the underlying connection stubs."""
    118 
    +        self.__actioncache_stub = None

  • buildgrid/server/actioncache/storage.py
    ... ... @@ -20,22 +20,39 @@ Action Cache
    20 20 
     Implements an in-memory action Cache
    21 21 
     """
    22 22
    23 
    +import logging
    23 24
    24 25 
     from ..referencestorage.storage import ReferenceCache
    25 26
    26 27
    27 28 
     class ActionCache(ReferenceCache):
    28 29
    30 
    +    def __init__(self, storage, max_cached_refs, allow_updates=True):
    31 
    +        super().__init__(storage, max_cached_refs, allow_updates)
    32 
    +
    33 
    +        self.__logger = logging.getLogger(__name__)
    34 
    +
    35 
    +    # --- Public API ---
    36 
    +
    29 37 
         def register_instance_with_server(self, instance_name, server):
    30 38 
             server.add_action_cache_instance(self, instance_name)
    31 39
    32 40 
         def get_action_result(self, action_digest):
    41 
    +        """Retrieves the cached result for an action."""
    33 42 
             key = self._get_key(action_digest)
    43 
    +
    34 44 
             return self.get_action_reference(key)
    35 45
    36 46 
         def update_action_result(self, action_digest, action_result):
    47 
    +        """Stores in cache a result for an action."""
    37 48 
             key = self._get_key(action_digest)
    49 
    +
    38 50 
             self.update_reference(key, action_result)
    39 51
    52 
    +        self.__logger.info("Result cached for action [%s/%s]",
    53 
    +                           action_digest.hash, action_digest.size_bytes)
    54 
    +
    55 
    +    # --- Private API ---
    56 
    +
    40 57 
         def _get_key(self, action_digest):
    41 58 
             return (action_digest.hash, action_digest.size_bytes)

  • buildgrid/utils.py
    ... ... @@ -17,7 +17,7 @@ from operator import attrgetter
    17 17 
     import os
    18 18 
     import socket
    19 19
    20 
    -from buildgrid.settings import HASH
    20 
    +from buildgrid.settings import HASH, HASH_LENGTH
    21 21 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    22 22
    23 23
    ... ... @@ -53,6 +53,27 @@ def create_digest(bytes_to_digest):
    53 53 
                                            size_bytes=len(bytes_to_digest))
    54 54
    55 55
    56 
    +def parse_digest(digest_string):
    57 
    +    """Creates a :obj:`Digest` from a digest string.
    58 
    +
    59 
    +    A digest string should alway be: ``{hash}/{size_bytes}``.
    60 
    +
    61 
    +    Args:
    62 
    +        digest_string (str): the digest string.
    63 
    +
    64 
    +    Returns:
    65 
    +        :obj:`Digest`: The :obj:`Digest` read from the string or None if
    66 
    +            `digest_string` is not a valid digest string.
    67 
    +    """
    68 
    +    digest_hash, digest_size = digest_string.split('/')
    69 
    +
    70 
    +    if len(digest_hash) == HASH_LENGTH and digest_size.isdigit():
    71 
    +        return remote_execution_pb2.Digest(hash=digest_hash,
    72 
    +                                           size_bytes=int(digest_size))
    73 
    +
    74 
    +    return None
    75 
    +
    76 
    +
    56 77 
     def read_file(file_path):
    57 78 
         """Loads raw file content in memory.
    58 79

  • docs/source/reference_cli.rst
    ... ... @@ -15,6 +15,27 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    15 15
    16 16 
     ----
    17 17
    18 
    +.. _invoking-bgd-action-cache:
    19 
    +
    20 
    +.. click:: buildgrid._app.commands.cmd_actioncache:cli
    21 
    +   :prog: bgd action-cache
    22 
    +
    23 
    +----
    24 
    +
    25 
    +.. _invoking-bgd-action-cache-get:
    26 
    +
    27 
    +.. click:: buildgrid._app.commands.cmd_actioncache:get
    28 
    +   :prog: bgd action-cache get
    29 
    +
    30 
    +----
    31 
    +
    32 
    +.. _invoking-bgd-action-cache-update:
    33 
    +
    34 
    +.. click:: buildgrid._app.commands.cmd_actioncache:update
    35 
    +   :prog: bgd action-cache update
    36 
    +
    37 
    +----
    38 
    +
    18 39 
     .. _invoking-bgd-bot:
    19 40
    20 41 
     .. click:: buildgrid._app.commands.cmd_bot:cli
    ... ... @@ -137,4 +158,4 @@ BuildGrid's Command Line Interface (CLI) reference documentation.
    137 158 
     .. _invoking-bgd-server-start:
    138 159
    139 160 
     .. click:: buildgrid._app.commands.cmd_server:start
    140 
    -   :prog: bgd server start
    \ No newline at end of file
    161 
    +   :prog: bgd server start

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]