GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][master] 3 commits: utils.py: Add a simple file writer helper

Title: GitLab

Martin Blanchard pushed to branch master at BuildGrid / buildgrid

Commits:

2 changed files:

Changes:

  • buildgrid/_app/bots/buildbox.py
    ... ... @@ -19,83 +19,126 @@ import tempfile
    19 19
    20 20 
     from google.protobuf import any_pb2
    21 21
    22 
    +from buildgrid.client.cas import upload
    22 23 
     from buildgrid._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
    23 24 
     from buildgrid._protos.google.bytestream import bytestream_pb2_grpc
    24 
    -from buildgrid.utils import parse_to_pb2_from_fetch
    25 
    +from buildgrid.utils import read_file, write_file, parse_to_pb2_from_fetch
    25 26
    26 27
    27 28 
     def work_buildbox(context, lease):
    29 
    +    """Executes a lease for a build action, using buildbox.
    30 
    +    """
    31 
    +
    32 
    +    stub_bytestream = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
    33 
    +    local_cas_directory = context.local_cas
    28 34 
         logger = context.logger
    29 35
    30 
    -    action_digest_any = lease.payload
    31 36 
         action_digest = remote_execution_pb2.Digest()
    32 
    -    action_digest_any.Unpack(action_digest)
    37 
    +    lease.payload.Unpack(action_digest)
    38 
    +
    39 
    +    action = parse_to_pb2_from_fetch(remote_execution_pb2.Action(),
    40 
    +                                     stub_bytestream, action_digest)
    33 41
    34 
    -    stub = bytestream_pb2_grpc.ByteStreamStub(context.cas_channel)
    42 
    +    command = parse_to_pb2_from_fetch(remote_execution_pb2.Command(),
    43 
    +                                      stub_bytestream, action.command_digest)
    35 44
    36 
    -    action = remote_execution_pb2.Action()
    37 
    -    parse_to_pb2_from_fetch(action, stub, action_digest)
    45 
    +    environment = dict()
    46 
    +    for variable in command.environment_variables:
    47 
    +        if variable.name not in ['PWD']:
    48 
    +            environment[variable.name] = variable.value
    38 49
    39 
    -    casdir = context.local_cas
    40 
    -    remote_command = remote_execution_pb2.Command()
    41 
    -    parse_to_pb2_from_fetch(remote_command, stub, action.command_digest)
    50 
    +    if command.working_directory:
    51 
    +        working_directory = command.working_directory
    52 
    +    else:
    53 
    +        working_directory = '/'
    42 54
    43 
    -    environment = dict((x.name, x.value) for x in remote_command.environment_variables)
    44 55 
         logger.debug("command hash: {}".format(action.command_digest.hash))
    45 56 
         logger.debug("vdir hash: {}".format(action.input_root_digest.hash))
    46 
    -    logger.debug("\n{}".format(' '.join(remote_command.arguments)))
    47 
    -
    48 
    -    # Input hash must be written to disk for buildbox.
    49 
    -    os.makedirs(os.path.join(casdir, 'tmp'), exist_ok=True)
    50 
    -    with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as input_digest_file:
    51 
    -        with open(input_digest_file.name, 'wb') as f:
    52 
    -            f.write(action.input_root_digest.SerializeToString())
    53 
    -            f.flush()
    54 
    -
    55 
    -        with tempfile.NamedTemporaryFile(dir=os.path.join(casdir, 'tmp')) as output_digest_file:
    56 
    -            command = ['buildbox',
    57 
    -                       '--remote={}'.format(context.remote_cas_url),
    58 
    -                       '--input-digest={}'.format(input_digest_file.name),
    59 
    -                       '--output-digest={}'.format(output_digest_file.name),
    60 
    -                       '--local={}'.format(casdir)]
    57 
    +    logger.debug("\n{}".format(' '.join(command.arguments)))
    58 
    +
    59 
    +    os.makedirs(os.path.join(local_cas_directory, 'tmp'), exist_ok=True)
    60 
    +    os.makedirs(context.fuse_dir, exist_ok=True)
    61 
    +
    62 
    +    with tempfile.NamedTemporaryFile(dir=os.path.join(local_cas_directory, 'tmp')) as input_digest_file:
    63 
    +        # Input hash must be written to disk for BuildBox
    64 
    +        write_file(input_digest_file.name, action.input_root_digest.SerializeToString())
    65 
    +
    66 
    +        with tempfile.NamedTemporaryFile(dir=os.path.join(local_cas_directory, 'tmp')) as output_digest_file:
    67 
    +            command_line = ['buildbox',
    68 
    +                            '--remote={}'.format(context.remote_cas_url),
    69 
    +                            '--input-digest={}'.format(input_digest_file.name),
    70 
    +                            '--output-digest={}'.format(output_digest_file.name),
    71 
    +                            '--chdir={}'.format(working_directory),
    72 
    +                            '--local={}'.format(local_cas_directory)]
    61 73
    62 74 
                 if context.cas_client_key:
    63 
    -                command.append('--client-key={}'.format(context.cas_client_key))
    75 
    +                command_line.append('--client-key={}'.format(context.cas_client_key))
    64 76 
                 if context.cas_client_cert:
    65 
    -                command.append('--client-cert={}'.format(context.cas_client_cert))
    77 
    +                command_line.append('--client-cert={}'.format(context.cas_client_cert))
    66 78 
                 if context.cas_server_cert:
    67 
    -                command.append('--server-cert={}'.format(context.cas_server_cert))
    68 
    -
    69 
    -            if 'PWD' in environment and environment['PWD']:
    70 
    -                command.append('--chdir={}'.format(environment['PWD']))
    79 
    +                command_line.append('--server-cert={}'.format(context.cas_server_cert))
    71 80
    72 
    -            command.append(context.fuse_dir)
    73 
    -            command.extend(remote_command.arguments)
    81 
    +            command_line.append(context.fuse_dir)
    82 
    +            command_line.extend(command.arguments)
    74 83
    75 
    -            logger.debug(' '.join(command))
    84 
    +            logger.debug(' '.join(command_line))
    76 85 
                 logger.debug("Input root digest:\n{}".format(action.input_root_digest))
    77 86 
                 logger.info("Launching process")
    78 87
    79 
    -            proc = subprocess.Popen(command,
    80 
    -                                    stdin=subprocess.PIPE,
    81 
    -                                    stdout=subprocess.PIPE)
    82 
    -            proc.communicate()
    88 
    +            command_line = subprocess.Popen(command_line,
    89 
    +                                            stdin=subprocess.PIPE,
    90 
    +                                            stdout=subprocess.PIPE)
    91 
    +            # TODO: Should return the stdout and stderr to the user.
    92 
    +            command_line.communicate()
    83 93
    84 
    -            output_root_digest = remote_execution_pb2.Digest()
    85 
    -            with open(output_digest_file.name, 'rb') as f:
    86 
    -                output_root_digest.ParseFromString(f.read())
    87 
    -            logger.debug("Output root digest: {}".format(output_root_digest))
    94 
    +            output_digest = remote_execution_pb2.Digest()
    95 
    +            output_digest.ParseFromString(read_file(output_digest_file.name))
    88 96
    89 
    -            if len(output_root_digest.hash) < 64:
    97 
    +            logger.debug("Output root digest: {}".format(output_digest))
    98 
    +
    99 
    +            if len(output_digest.hash) < 64:
    90 100 
                     logger.warning("Buildbox command failed - no output root digest present.")
    91 
    -            output_file = remote_execution_pb2.OutputDirectory(tree_digest=output_root_digest)
    92 101
    93 
    -    action_result = remote_execution_pb2.ActionResult()
    94 
    -    action_result.output_directories.extend([output_file])
    102 
    +            # TODO: Have BuildBox helping us creating the Tree instance here
    103 
    +            # See https://gitlab.com/BuildStream/buildbox/issues/7 for details
    104 
    +            output_tree = _cas_tree_maker(stub_bytestream, output_digest)
    105 
    +
    106 
    +            with upload(context.cas_channel) as cas:
    107 
    +                output_tree_digest = cas.send_message(output_tree)
    95 108
    96 
    -    action_result_any = any_pb2.Any()
    97 
    -    action_result_any.Pack(action_result)
    109 
    +            output_directory = remote_execution_pb2.OutputDirectory()
    110 
    +            output_directory.tree_digest.CopyFrom(output_tree_digest)
    111 
    +            output_directory.path = os.path.relpath(working_directory, start='/')
    98 112
    99 
    -    lease.result.CopyFrom(action_result_any)
    113 
    +            action_result = remote_execution_pb2.ActionResult()
    114 
    +            action_result.output_directories.extend([output_directory])
    115 
    +
    116 
    +            action_result_any = any_pb2.Any()
    117 
    +            action_result_any.Pack(action_result)
    118 
    +
    119 
    +            lease.result.CopyFrom(action_result_any)
    100 120
    101 121 
         return lease
    122 
    +
    123 
    +
    124 
    +def _cas_tree_maker(stub_bytestream, directory_digest):
    125 
    +    # Generates and stores a Tree for a given Directory. This is very inefficient
    126 
    +    # and only temporary. See https://gitlab.com/BuildStream/buildbox/issues/7.
    127 
    +    output_tree = remote_execution_pb2.Tree()
    128 
    +
    129 
    +    def list_directories(parent_directory):
    130 
    +        directory_list = list()
    131 
    +        for directory_node in parent_directory.directories:
    132 
    +            directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
    133 
    +                                                stub_bytestream, directory_node.digest)
    134 
    +            directory_list.extend(list_directories(directory))
    135 
    +            directory_list.append(directory)
    136 
    +
    137 
    +        return directory_list
    138 
    +
    139 
    +    root_directory = parse_to_pb2_from_fetch(remote_execution_pb2.Directory(),
    140 
    +                                             stub_bytestream, directory_digest)
    141 
    +    output_tree.children.extend(list_directories(root_directory))
    142 
    +    output_tree.root.CopyFrom(root_directory)
    143 
    +
    144 
    +    return output_tree

  • buildgrid/utils.py
    ... ... @@ -304,6 +304,21 @@ def read_file(file_path):
    304 304 
             return byte_file.read()
    305 305
    306 306
    307 
    +def write_file(file_path, content):
    308 
    +    """Dumps raw memory content to a file.
    309 
    +
    310 
    +    Args:
    311 
    +        file_path (str): path to the target file.
    312 
    +        content (bytes): raw file's content.
    313 
    +
    314 
    +    Raises:
    315 
    +        OSError: If `file_path` does not exist or is not writable.
    316 
    +    """
    317 
    +    with open(file_path, 'wb') as byte_file:
    318 
    +        byte_file.write(content)
    319 
    +        byte_file.flush()
    320 
    +
    321 
    +
    307 322 
     def output_file_maker(file_path, input_path, cas=None):
    308 323 
         """Creates an :obj:`OutputFile` from a local file and possibly upload it.
    309 324

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]