GNOME.org
 

[Notes] [Git][BuildStream/buildstream][jmac/remote_execution_client] 13 commits: cascache.py: Update to new protocol buffers spec

Title: GitLab

Jim MacArthur pushed to branch jmac/remote_execution_client at BuildStream / buildstream

Commits:

12 changed files:

Changes:

  • buildstream/_artifactcache/cascache.py
    ... ... @@ -403,11 +403,11 @@ class CASCache(ArtifactCache):
    403 403 
             # Check whether ref is already on the server in which case
    404 404 
             # there is no need to push the artifact
    405 405 
             try:
    406 
    -            request = buildstream_pb2.GetArtifactRequest()
    406 
    +            request = buildstream_pb2.GetReferenceRequest()
    407 407 
                 request.key = ref
    408 
    -            response = remote.artifact_cache.GetArtifact(request)
    408 
    +            response = remote.ref_storage.GetReference(request)
    409 409
    410 
    -            if response.artifact.hash == tree.hash and response.artifact.size_bytes == tree.size_bytes:
    410 
    +            if response.digest.hash == tree.hash and response.digest.size_bytes == tree.size_bytes:
    411 411 
                     # ref is already on the server with the same tree
    412 412 
                     return True
    413 413

  • buildstream/_loader/loader.py
    ... ... @@ -446,6 +446,7 @@ class Loader():
    446 446 
                                        _yaml.node_get(node, list, Symbol.ENV_NOCACHE, default_value=[]),
    447 447 
                                        _yaml.node_get(node, Mapping, Symbol.PUBLIC, default_value={}),
    448 448 
                                        _yaml.node_get(node, Mapping, Symbol.SANDBOX, default_value={}),
    449 
    +                                   _yaml.node_get(node, Mapping, Symbol.REMOTE_EXECUTION, default_value={}),
    449 450 
                                        element_kind == 'junction')
    450 451
    451 452 
             # Cache it now, make sure it's already there before recursing

  • buildstream/_loader/metaelement.py
    ... ... @@ -39,7 +39,7 @@ class MetaElement():
    39 39 
         #    first_pass: The element is to be loaded with first pass configuration (junction)
    40 40 
         #
    41 41 
         def __init__(self, project, name, kind, provenance, sources, config,
    42 
    -                 variables, environment, env_nocache, public, sandbox,
    42 
    +                 variables, environment, env_nocache, public, sandbox, remote_execution,
    43 43 
                      first_pass):
    44 44 
             self.project = project
    45 45 
             self.name = name
    ... ... @@ -52,6 +52,7 @@ class MetaElement():
    52 52 
             self.env_nocache = env_nocache
    53 53 
             self.public = public
    54 54 
             self.sandbox = sandbox
    55 
    +        self.remote_execution = remote_execution
    55 56 
             self.build_dependencies = []
    56 57 
             self.dependencies = []
    57 58 
             self.first_pass = first_pass

  • buildstream/_loader/types.py
    ... ... @@ -39,6 +39,7 @@ class Symbol():
    39 39 
         DIRECTORY = "directory"
    40 40 
         JUNCTION = "junction"
    41 41 
         SANDBOX = "sandbox"
    42 
    +    REMOTE_EXECUTION = "remote-execution"
    42 43
    43 44
    44 45 
     # Dependency()

  • buildstream/_platform/linux.py
    ... ... @@ -24,6 +24,7 @@ from .. import utils
    24 24 
     from .._artifactcache.cascache import CASCache
    25 25 
     from .._message import Message, MessageType
    26 26 
     from ..sandbox import SandboxBwrap
    27 
    +from ..sandbox import SandboxRemote
    27 28
    28 29 
     from . import Platform
    29 30

  • buildstream/_project.py
    ... ... @@ -127,6 +127,7 @@ class Project():
    127 127
    128 128 
             self.artifact_cache_specs = None
    129 129 
             self._sandbox = None
    130 
    +        self._remote_execution = None
    130 131 
             self._splits = None
    131 132
    132 133 
             self._context.add_project(self)
    ... ... @@ -458,7 +459,8 @@ class Project():
    458 459 
                 'aliases', 'name',
    459 460 
                 'artifacts', 'options',
    460 461 
                 'fail-on-overlap', 'shell',
    461 
    -            'ref-storage', 'sandbox', 'mirrors'
    462 
    +            'ref-storage', 'sandbox',
    463 
    +            'remote-execution', 'mirrors'
    462 464 
             ])
    463 465
    464 466 
             #
    ... ... @@ -476,6 +478,9 @@ class Project():
    476 478 
             # Load sandbox configuration
    477 479 
             self._sandbox = _yaml.node_get(config, Mapping, 'sandbox')
    478 480
    481 
    +        # Load remote execution configuration
    482 
    +        self._remote_execution = _yaml.node_get(config, Mapping, 'remote-execution')
    483 
    +
    479 484 
             # Load project split rules
    480 485 
             self._splits = _yaml.node_get(config, Mapping, 'split-rules')
    481 486

  • buildstream/buildelement.py
    ... ... @@ -155,6 +155,9 @@ class BuildElement(Element):
    155 155 
                 command_dir = build_root
    156 156 
             sandbox.set_work_directory(command_dir)
    157 157
    158 
    +        # Tell sandbox which directory is preserved in the finished artifact
    159 
    +        sandbox.set_output_directory(install_root)
    160 
    +
    158 161 
             # Setup environment
    159 162 
             sandbox.set_environment(self.get_environment())
    160 163

  • buildstream/element.py
    ... ... @@ -94,6 +94,7 @@ from . import _signals
    94 94 
     from . import _site
    95 95 
     from ._platform import Platform
    96 96 
     from .sandbox._config import SandboxConfig
    97 
    +from .sandbox._sandboxremote import SandboxRemote
    97 98
    98 99 
     from .storage.directory import Directory
    99 100 
     from .storage._filebaseddirectory import FileBasedDirectory
    ... ... @@ -249,6 +250,9 @@ class Element(Plugin):
    249 250 
             # Extract Sandbox config
    250 251 
             self.__sandbox_config = self.__extract_sandbox_config(meta)
    251 252
    253 
    +        # Extract remote execution URL
    254 
    +        self.__remote_execution_url = self.__extract_remote_execution_config(meta)
    255 
    +
    252 256 
         def __lt__(self, other):
    253 257 
             return self.name < other.name
    254 258
    ... ... @@ -1546,6 +1550,8 @@ class Element(Plugin):
    1546 1550 
                     finally:
    1547 1551 
                         if collect is not None:
    1548 1552 
                             try:
    1553 
    +                            # Sandbox will probably have replaced its virtual directory, so get it again
    1554 
    +                            sandbox_vroot = sandbox.get_virtual_directory()
    1549 1555 
                                 collectvdir = sandbox_vroot.descend(collect.lstrip(os.sep).split(os.sep))
    1550 1556 
                             except VirtualDirectoryError:
    1551 1557 
                                 # No collect directory existed
    ... ... @@ -2120,7 +2126,24 @@ class Element(Plugin):
    2120 2126 
             project = self._get_project()
    2121 2127 
             platform = Platform.get_platform()
    2122 2128
    2123 
    -        if directory is not None and os.path.exists(directory):
    2129 
    +        if self.__remote_execution_url is not None and self.BST_VIRTUAL_DIRECTORY:
    2130 
    +            if not self.__artifacts.has_push_remotes(element=self):
    2131 
    +                # Give an early warning if remote execution will not work
    2132 
    +                raise ElementError("Artifact {} is configured to use remote execution but has no push remotes. "
    2133 
    +                                   .format(self.name) +
    2134 
    +                                   "The remote artifact server(s) may not be correctly configured or contactable.")
    2135 
    +
    2136 
    +            self.info("Using a remote 'sandbox' for artifact {}".format(self.name))
    2137 
    +            sandbox = SandboxRemote(context, project,
    2138 
    +                                              directory,
    2139 
    +                                              stdout=stdout,
    2140 
    +                                              stderr=stderr,
    2141 
    +                                              config=config,
    2142 
    +                                              server_url=self.__remote_execution_url,
    2143 
    +                                              allow_real_directory=False)
    2144 
    +            yield sandbox
    2145 
    +        elif directory is not None and os.path.exists(directory):
    2146 
    +            self.info("Using a local sandbox for artifact {}".format(self.name))
    2124 2147 
                 sandbox = platform.create_sandbox(context, project,
    2125 2148 
                                                   directory,
    2126 2149 
                                                   stdout=stdout,
    ... ... @@ -2292,6 +2315,24 @@ class Element(Plugin):
    2292 2315 
             return SandboxConfig(self.node_get_member(sandbox_config, int, 'build-uid'),
    2293 2316 
                                  self.node_get_member(sandbox_config, int, 'build-gid'))
    2294 2317
    2318 
    +    def __extract_remote_execution_config(self, meta):
    2319 
    +        project = self._get_project()
    2320 
    +        project.ensure_fully_loaded()
    2321 
    +        rexec_config = _yaml.node_chain_copy(project._remote_execution)
    2322 
    +
    2323 
    +        # The default config is already composited with the project overrides
    2324 
    +        rexec_defaults = _yaml.node_get(self.__defaults, Mapping, 'remote-execution', default_value={})
    2325 
    +        rexec_defaults = _yaml.node_chain_copy(rexec_defaults)
    2326 
    +
    2327 
    +        _yaml.composite(rexec_config, rexec_defaults)
    2328 
    +        _yaml.composite(rexec_config, meta.remote_execution)
    2329 
    +        _yaml.node_final_assertions(rexec_config)
    2330 
    +
    2331 
    +        # Rexec config, unlike others, has fixed members so we should validate them
    2332 
    +        _yaml.node_validate(rexec_config, ['url'])
    2333 
    +
    2334 
    +        return self.node_get_member(rexec_config, str, 'url')
    2335 
    +
    2295 2336 
         # This makes a special exception for the split rules, which
    2296 2337 
         # elements may extend but whos defaults are defined in the project.
    2297 2338 
         #

  • buildstream/plugins/elements/autotools.py
    ... ... @@ -57,7 +57,7 @@ from buildstream import BuildElement
    57 57
    58 58 
     # Element implementation for the 'autotools' kind.
    59 59 
     class AutotoolsElement(BuildElement):
    60 
    -    pass
    60 
    +    BST_VIRTUAL_DIRECTORY = True
    61 61
    62 62
    63 63 
     # Plugin entry point

  • buildstream/sandbox/__init__.py
    ... ... @@ -20,3 +20,4 @@
    20 20 
     from .sandbox import Sandbox, SandboxFlags
    21 21 
     from ._sandboxchroot import SandboxChroot
    22 22 
     from ._sandboxbwrap import SandboxBwrap
    23 
    +from ._sandboxremote import SandboxRemote

  • buildstream/sandbox/_sandboxremote.py
    1 
    +#!/usr/bin/env python3
    2 
    +#
    3 
    +#  Copyright (C) 2018 Codethink Limited
    4 
    +#
    5 
    +#  This program is free software; you can redistribute it and/or
    6 
    +#  modify it under the terms of the GNU Lesser General Public
    7 
    +#  License as published by the Free Software Foundation; either
    8 
    +#  version 2 of the License, or (at your option) any later version.
    9 
    +#
    10 
    +#  This library is distributed in the hope that it will be useful,
    11 
    +#  but WITHOUT ANY WARRANTY; without even the implied warranty of
    12 
    +#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
    13 
    +#  Lesser General Public License for more details.
    14 
    +#
    15 
    +#  You should have received a copy of the GNU Lesser General Public
    16 
    +#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
    17 
    +#
    18 
    +#  Authors:
    19 
    +#        Jim MacArthur <jim macarthur codethink co uk>
    20 
    +
    21 
    +import os
    22 
    +import re
    23 
    +import time
    24 
    +
    25 
    +import grpc
    26 
    +
    27 
    +from . import Sandbox
    28 
    +from ..storage._filebaseddirectory import FileBasedDirectory
    29 
    +from ..storage._casbaseddirectory import CasBasedDirectory
    30 
    +from .._protos.build.bazel.remote.execution.v2 import remote_execution_pb2, remote_execution_pb2_grpc
    31 
    +from .._protos.google.longrunning import operations_pb2, operations_pb2_grpc
    32 
    +
    33 
    +from .._artifactcache.cascache import CASCache
    34 
    +
    35 
    +
    36 
    +class SandboxError(Exception):
    37 
    +    pass
    38 
    +
    39 
    +
    40 
    +# SandboxRemote()
    41 
    +#
    42 
    +# This isn't really a sandbox, it's a stub which sends all the source
    43 
    +# to a remote server and retrieves the results from it.
    44 
    +#
    45 
    +class SandboxRemote(Sandbox):
    46 
    +
    47 
    +    def __init__(self, *args, **kwargs):
    48 
    +        super().__init__(*args, **kwargs)
    49 
    +        self.cascache = None
    50 
    +        self.server_url = kwargs['server_url']
    51 
    +        # Check the format of the url ourselves to save the user from
    52 
    +        # whatever error messages grpc will produce
    53 
    +        m = re.match('^(.+):(\d+)$', self.server_url)
    54 
    +        if m is None:
    55 
    +            raise SandboxError("Configured remote URL '{}' does not match the expected layout. " +
    56 
    +                               "It should be of the form <protocol>://<domain name>:<port>."
    57 
    +                               .format(server_url))
    58 
    +
    59 
    +
    60 
    +    def _get_cascache(self):
    61 
    +        if self.cascache is None:
    62 
    +            self.cascache = CASCache(self._get_context())
    63 
    +            self.cascache.setup_remotes(use_config=True)
    64 
    +        return self.cascache
    65 
    +
    66 
    +    def __run_remote_command(self, cascache, command, input_root_digest, environment):
    67 
    +
    68 
    +        environment_variables = [ remote_execution_pb2.Command.
    69 
    +                                  EnvironmentVariable(name=k, value=v)
    70 
    +                                  for (k,v) in environment.items() ]
    71 
    +
    72 
    +        # Create and send the Command object.
    73 
    +        remote_command = remote_execution_pb2.Command(arguments=command, environment_variables=environment_variables,
    74 
    +                                                      output_files=[],
    75 
    +                                                      output_directories=[self._output_directory],
    76 
    +                                                      platform=None)
    77 
    +        command_digest = cascache.add_object(buffer=remote_command.SerializeToString())
    78 
    +        command_ref = 'worker-command/{}'.format(command_digest.hash)
    79 
    +        cascache.set_ref(command_ref, command_digest)
    80 
    +
    81 
    +        command_push_successful = cascache.push_refs([command_ref], self._get_project(), may_have_dependencies=False)
    82 
    +        if not command_push_successful and not cascache.verify_key_pushed(command_ref, self._get_project()):
    83 
    +            # Command push failed
    84 
    +            return None
    85 
    +
    86 
    +        # Create and send the action.
    87 
    +
    88 
    +        action = remote_execution_pb2.Action(command_digest=command_digest,
    89 
    +                                             input_root_digest=input_root_digest,
    90 
    +                                             timeout=None,
    91 
    +                                             do_not_cache=True)
    92 
    +
    93 
    +        action_digest = cascache.add_object(buffer=action.SerializeToString())
    94 
    +        action_ref = 'worker-action/{}'.format(command_digest.hash)
    95 
    +        cascache.set_ref(action_ref, action_digest)
    96 
    +        action_push_successful = cascache.push_refs([action_ref], self._get_project(), may_have_dependencies=False)
    97 
    +
    98 
    +        if not action_push_successful and not cascache.verify_key_pushed(action_ref, self._get_project()):
    99 
    +            # Action push failed
    100 
    +            return None
    101 
    +
    102 
    +        # Next, try to create a communication channel to the BuildGrid server.
    103 
    +
    104 
    +        channel = grpc.insecure_channel(self.server_url)
    105 
    +        stub = remote_execution_pb2_grpc.ExecutionStub(channel)
    106 
    +        request = remote_execution_pb2.ExecuteRequest(instance_name='default',
    107 
    +                                                      action_digest=action_digest,
    108 
    +                                                      skip_cache_lookup=True)
    109 
    +
    110 
    +        operation_iterator = stub.Execute(request)
    111 
    +        if operation_iterator.code() != grpc.StatusCode.OK:
    112 
    +            raise SandboxError("GRPC Execute failed; is the remote system connected?")
    113 
    +        for operation in operation_iterator:
    114 
    +            if operation.done:
    115 
    +                break
    116 
    +            # TODO: Do we need a sleep here?
    117 
    +        return operation
    118 
    +
    119 
    +    def process_job_output(self, output_directories, output_files):
    120 
    +        # output_directories is an array of OutputDirectory objects.
    121 
    +        # output_files is an array of OutputFile objects.
    122 
    +        #
    123 
    +        # We only specify one output_directory, so it's an error
    124 
    +        # for there to be any output files or more than one directory at the moment.
    125 
    +
    126 
    +        if output_files:
    127 
    +            raise SandboxError("Output files were returned when we didn't request any.")
    128 
    +        elif len(output_directories) > 1:
    129 
    +            error_text = "More than one output directory was returned from the build server: {}"
    130 
    +            raise SandboxError(error_text.format(output_directories))
    131 
    +        elif len(output_directories) < 1:
    132 
    +            error_text = "No output directory was returned from the build server."
    133 
    +            raise SandboxError(error_test)
    134 
    +
    135 
    +        digest = output_directories[0].tree_digest
    136 
    +        if digest is None or digest.hash is None or digest.hash == "":
    137 
    +            raise SandboxError("Output directory structure had no digest attached.")
    138 
    +
    139 
    +        # Now do a pull to ensure we have the necessary parts.
    140 
    +        cascache = self._get_cascache()
    141 
    +        cascache.pull_key(digest.hash, digest.size_bytes, self._get_project())
    142 
    +        path_components = os.path.split(self._output_directory)
    143 
    +
    144 
    +        # Now what we have is a digest for the output. Once we return, the calling process will
    145 
    +        # attempt to descend into our directory and find that directory, so we need to overwrite
    146 
    +        # that.
    147 
    +
    148 
    +        if not path_components:
    149 
    +            # The artifact wants the whole directory; we could just return the returned hash in its
    150 
    +            # place, but we don't have a means to do that yet.
    151 
    +            raise SandboxError("Unimplemented: Output directory is empty or equal to the sandbox root.")
    152 
    +
    153 
    +        # At the moment, we will get the whole directory back in the first directory argument and we need
    154 
    +        # to replace the sandbox's virtual directory with that. Creating a new virtual directory object
    155 
    +        # from another hash will be interesting, though...
    156 
    +
    157 
    +        new_dir = CasBasedDirectory(self._get_context(), ref=digest)
    158 
    +        self.set_virtual_directory(new_dir)
    159 
    +
    160 
    +    def run(self, command, flags, *, cwd=None, env=None):
    161 
    +        # Upload sources
    162 
    +        upload_vdir = self.get_virtual_directory()
    163 
    +
    164 
    +        if isinstance(upload_vdir, FileBasedDirectory):
    165 
    +            # Make a new temporary directory to put source in
    166 
    +            upload_vdir = CasBasedDirectory(self._get_context(), ref=None)
    167 
    +            upload_vdir.import_files(self.get_virtual_directory()._get_underlying_directory())
    168 
    +
    169 
    +
    170 
    +        # Now, push that key (without necessarily needing a ref) to the remote.
    171 
    +        cascache = self._get_cascache()
    172 
    +
    173 
    +        ref = 'worker-source/{}'.format(upload_vdir.ref.hash)
    174 
    +        upload_vdir._save(ref)
    175 
    +        source_push_successful = cascache.push_refs([ref], self._get_project())
    176 
    +
    177 
    +        # Set up environment and PWD
    178 
    +        if env is None:
    179 
    +            env = self._get_environment()
    180 
    +        if 'PWD' not in env:
    181 
    +            env['PWD'] = self._get_work_directory()
    182 
    +
    183 
    +        # We want command args as a list of strings
    184 
    +        if isinstance(command, str):
    185 
    +            command = [command]
    186 
    +
    187 
    +        # Now transmit the command to execute
    188 
    +        if source_push_successful or cascache.verify_key_pushed(ref, self._get_project()):
    189 
    +            response = self.__run_remote_command(cascache, command, upload_vdir.ref, env)
    190 
    +
    191 
    +            if response is None:
    192 
    +                # Failure of remote execution, usually due to an error in BuildStream
    193 
    +                # NB This error could be raised in __run_remote_command
    194 
    +                raise SandboxError("No response returned from server")
    195 
    +
    196 
    +            assert(response.HasField("error") or response.HasField("response"))
    197 
    +
    198 
    +            if response.HasField("error"):
    199 
    +                # A normal error during the build
    200 
    +                error_message = response.error.message
    201 
    +                # response.error also contains 'details' (iterator of Any) which we ignore at the moment.
    202 
    +                return response.error.code
    203 
    +            else:
    204 
    +
    205 
    +                # At the moment, response can either be an
    206 
    +                # ExecutionResponse containing an ActionResult, or an
    207 
    +                # ActionResult directly.
    208 
    +                executeResponse = remote_execution_pb2.ExecuteResponse()
    209 
    +                if response.response.Is(executeResponse.DESCRIPTOR):
    210 
    +                    # Unpack ExecuteResponse and set response to its response
    211 
    +                    response.response.Unpack(executeResponse)
    212 
    +                    response = executeResponse
    213 
    +
    214 
    +                actionResult = remote_execution_pb2.ActionResult()
    215 
    +                if response.response.Is(actionResult.DESCRIPTOR):
    216 
    +                    response.response.Unpack(actionResult)
    217 
    +                    self.process_job_output(actionResult.output_directories, actionResult.output_files)
    218 
    +                else:
    219 
    +                    raise SandboxError("Received unknown message from server (expected ExecutionResponse).")
    220 
    +        else:
    221 
    +            raise SandboxError("Failed to verify that source has been pushed to the remote artifact cache.")
    222 
    +        return 0

  • buildstream/sandbox/sandbox.py
    ... ... @@ -99,9 +99,11 @@ class Sandbox():
    99 99 
             self.__stdout = kwargs['stdout']
    100 100 
             self.__stderr = kwargs['stderr']
    101 101
    102 
    -        # Setup the directories. Root should be available to subclasses, hence
    103 
    -        # being single-underscore. The others are private to this class.
    102 
    +        # Setup the directories. Root and output_directory should be
    103 
    +        # available to subclasses, hence being single-underscore. The
    104 
    +        # others are private to this class.
    104 105 
             self._root = os.path.join(directory, 'root')
    106 
    +        self._output_directory = None
    105 107 
             self.__directory = directory
    106 108 
             self.__scratch = os.path.join(self.__directory, 'scratch')
    107 109 
             for directory_ in [self._root, self.__scratch]:
    ... ... @@ -144,11 +146,29 @@ class Sandbox():
    144 146 
                     self._vdir = FileBasedDirectory(self._root)
    145 147 
             return self._vdir
    146 148
    149 
    +    def set_virtual_directory(self, vdir):
    150 
    +        """ Sets virtual directory. Useful after remote execution
    151 
    +        has rewritten the working directory. """
    152 
    +        self._vdir = vdir
    153 
    +
    154 
    +    def get_virtual_toplevel_directory(self):
    155 
    +        """Fetches the sandbox's toplevel directory
    156 
    +
    157 
    +        The toplevel directory contains 'root', 'scratch' and later
    158 
    +        'artifact' where output is copied to.
    159 
    +
    160 
    +        Returns:
    161 
    +           (str): The sandbox toplevel directory
    162 
    +
    163 
    +        """
    164 
    +        # For now, just create a new Directory every time we're asked
    165 
    +        return FileBasedDirectory(self.__directory)
    166 
    +
    147 167 
         def set_environment(self, environment):
    148 168 
             """Sets the environment variables for the sandbox
    149 169
    150 170 
             Args:
    151 
    -           directory (dict): The environment variables to use in the sandbox
    171 
    +           environment (dict): The environment variables to use in the sandbox
    152 172 
             """
    153 173 
             self.__env = environment
    154 174
    ... ... @@ -160,6 +180,15 @@ class Sandbox():
    160 180 
             """
    161 181 
             self.__cwd = directory
    162 182
    183 
    +    def set_output_directory(self, directory):
    184 
    +        """Sets the output directory - the directory which is preserved
    185 
    +        as an artifact after assembly.
    186 
    +
    187 
    +        Args:
    188 
    +           directory (str): An absolute path within the sandbox
    189 
    +        """
    190 
    +        self._output_directory = directory
    191 
    +
    163 192 
         def mark_directory(self, directory, *, artifact=False):
    164 193 
             """Marks a sandbox directory and ensures it will exist
    165 194

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]