GNOME.org
 

[Notes] [Git][BuildGrid/buildgrid][finn/74-operation-cancelation] 5 commits: Add Device class.

Title: GitLab

finn pushed to branch finn/74-operation-cancelation at BuildGrid / buildgrid

Commits:

6 changed files:

Changes:

  • buildgrid/bot/device.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +class Device:
    17 
    +
    18 
    +    def __init__(self, properties=None):
    19 
    +        """ Creates devices available to the worker
    20 
    +        The first device is know as the Primary Device - the revice which
    21 
    +        is running a bit and responsible to actually executing commands.
    22 
    +        All other devices are known as Attatched Devices and must be controlled
    23 
    +        by the Primary Device.
    24 
    +        """
    25 
    +
    26 
    +        self.__properties = {}
    27 
    +        self.__name = str(uuid.uuid4())
    28 
    +
    29 
    +        if properties:
    30 
    +            for k, v in properties.items():
    31 
    +                if k == 'os':
    32 
    +                    self.__properties[k] = v
    33 
    +
    34 
    +                elif k == 'docker':
    35 
    +                    if v not in ('True', 'False'):
    36 
    +                        raise ValueError('Value not supported: [{}]'.format(v))
    37 
    +                    self.__properties[k] = v
    38 
    +
    39 
    +                else:
    40 
    +                    raise KeyError('Key not supported: [{}]'.format(k))
    41 
    +
    42 
    +    @property
    43 
    +    def name(self):
    44 
    +        return self.__name
    45 
    +
    46 
    +    @property
    47 
    +    def properties(self):
    48 
    +        return self.__properties
    49 
    +
    50 
    +    def get_pb2(self):
    51 
    +        device = worker_pb2.Device(handle=self._name)
    52 
    +        property_message = worker_pb2.Device.Property()
    53 
    +        for k, v in self._properties.items():
    54 
    +            property_message.key = k
    55 
    +            property_message.value = v
    56 
    +            device.properties.extend([property_message])
    57 
    +        return device

  • buildgrid/bot/bot_interface.py → buildgrid/bot/interface.py

  • buildgrid/bot/bot_session.py → buildgrid/bot/session.py
    ... ... @@ -18,7 +18,7 @@
    18 18
    19 19 
     """
    20 20 
     Bot Session
    21 
    -====
    21 
    +===========
    22 22
    23 23 
     Allows connections
    24 24 
     """
    ... ... @@ -52,25 +52,8 @@ class BotStatus(Enum):
    52 52 
         BOT_TERMINATING = bots_pb2.BotStatus.Value('BOT_TERMINATING')
    53 53
    54 54
    55 
    -class LeaseState(Enum):
    56 
    -    # Default value.
    57 
    -    LEASE_STATE_UNSPECIFIED = bots_pb2.LeaseState.Value('LEASE_STATE_UNSPECIFIED')
    58 
    -
    59 
    -    # The server expects the bot to accept this lease.
    60 
    -    PENDING = bots_pb2.LeaseState.Value('PENDING')
    61 
    -
    62 
    -    # The bot has accepted this lease.
    63 
    -    ACTIVE = bots_pb2.LeaseState.Value('ACTIVE')
    64 
    -
    65 
    -    # The bot is no longer leased.
    66 
    -    COMPLETED = bots_pb2.LeaseState.Value('COMPLETED')
    67 
    -
    68 
    -    # The bot should immediately release all resources associated with the lease.
    69 
    -    CANCELLED = bots_pb2.LeaseState.Value('CANCELLED')
    70 
    -
    71 
    -
    72 55 
     class BotSession:
    73 
    -    def __init__(self, parent, interface):
    56 
    +    def __init__(self, parent, interface, worker):
    74 57 
             """ Unique bot ID within the farm used to identify this bot
    75 58 
             Needs to be human readable.
    76 59 
             All prior sessions with bot_id of same ID are invalidated.
    ... ... @@ -79,23 +62,20 @@ class BotSession:
    79 62 
             """
    80 63
    81 64 
             self.logger = logging.getLogger(__name__)
    82 
    -
    83 
    -        self._bot_id = '{}.{}'.format(parent, platform.node())
    84 65 
             self._context = None
    66 
    +
    67 
    +        self._worker = worker
    85 68 
             self._interface = interface
    86 69 
             self._leases = {}
    87 
    -        self._name = None
    88 70 
             self._parent = parent
    89 71 
             self._status = BotStatus.OK.value
    90 
    -        self._work = None
    91 
    -        self._worker = None
    72 
    +
    73 
    +        self.__bot_id = '{}.{}'.format(parent, platform.node())
    74 
    +        self.__name = None
    92 75
    93 76 
         @property
    94 77 
         def bot_id(self):
    95 
    -        return self._bot_id
    96 
    -
    97 
    -    def add_worker(self, worker):
    98 
    -        self._worker = worker
    78 
    +        return self.__bot_id
    99 79
    100 80 
         def create_bot_session(self, work, context=None):
    101 81 
             self.logger.debug("Creating bot session")
    ... ... @@ -103,15 +83,15 @@ class BotSession:
    103 83 
             self._context = context
    104 84
    105 85 
             session = self._interface.create_bot_session(self._parent, self.get_pb2())
    106 
    -        self._name = session.name
    86 
    +        self.__name = session.name
    107 87
    108 
    -        self.logger.info("Created bot session with name: [{}]".format(self._name))
    88 
    +        self.logger.info("Created bot session with name: [{}]".format(self.__name))
    109 89
    110 90 
             for lease in session.leases:
    111 91 
                 self._update_lease_from_server(lease)
    112 92
    113 93 
         def update_bot_session(self):
    114 
    -        self.logger.debug("Updating bot session: [{}]".format(self._bot_id))
    94 
    +        self.logger.debug("Updating bot session: [{}]".format(self.__bot_id))
    115 95 
             session = self._interface.update_bot_session(self.get_pb2())
    116 96 
             for k, v in list(self._leases.items()):
    117 97 
                 if v.state == LeaseState.COMPLETED.value:
    ... ... @@ -128,8 +108,8 @@ class BotSession:
    128 108 
             return bots_pb2.BotSession(worker=self._worker.get_pb2(),
    129 109 
                                        status=self._status,
    130 110 
                                        leases=leases,
    131 
    -                                   bot_id=self._bot_id,
    132 
    -                                   name=self._name)
    111 
    +                                   bot_id=self.__bot_id,
    112 
    +                                   name=self.__name)
    133 113
    134 114 
         def lease_completed(self, lease):
    135 115 
             lease.state = LeaseState.COMPLETED.value
    ... ... @@ -167,91 +147,3 @@ class BotSession:
    167 147
    168 148 
             self.logger.debug("Work complete: [{}]".format(lease.id))
    169 149 
             self.lease_completed(lease)
    170 
    -
    171 
    -
    172 
    -class Worker:
    173 
    -    def __init__(self, properties=None, configs=None):
    174 
    -        self.properties = {}
    175 
    -        self._configs = {}
    176 
    -        self._devices = []
    177 
    -
    178 
    -        if properties:
    179 
    -            for k, v in properties.items():
    180 
    -                if k == 'pool':
    181 
    -                    self.properties[k] = v
    182 
    -                else:
    183 
    -                    raise KeyError('Key not supported: [{}]'.format(k))
    184 
    -
    185 
    -        if configs:
    186 
    -            for k, v in configs.items():
    187 
    -                if k == 'DockerImage':
    188 
    -                    self.configs[k] = v
    189 
    -                else:
    190 
    -                    raise KeyError('Key not supported: [{}]'.format(k))
    191 
    -
    192 
    -    @property
    193 
    -    def configs(self):
    194 
    -        return self._configs
    195 
    -
    196 
    -    def add_device(self, device):
    197 
    -        self._devices.append(device)
    198 
    -
    199 
    -    def get_pb2(self):
    200 
    -        devices = [device.get_pb2() for device in self._devices]
    201 
    -        worker = worker_pb2.Worker(devices=devices)
    202 
    -        property_message = worker_pb2.Worker.Property()
    203 
    -        for k, v in self.properties.items():
    204 
    -            property_message.key = k
    205 
    -            property_message.value = v
    206 
    -            worker.properties.extend([property_message])
    207 
    -
    208 
    -        config_message = worker_pb2.Worker.Config()
    209 
    -        for k, v in self.properties.items():
    210 
    -            property_message.key = k
    211 
    -            property_message.value = v
    212 
    -            worker.configs.extend([config_message])
    213 
    -
    214 
    -        return worker
    215 
    -
    216 
    -
    217 
    -class Device:
    218 
    -    def __init__(self, properties=None):
    219 
    -        """ Creates devices available to the worker
    220 
    -        The first device is know as the Primary Device - the revice which
    221 
    -        is running a bit and responsible to actually executing commands.
    222 
    -        All other devices are known as Attatched Devices and must be controlled
    223 
    -        by the Primary Device.
    224 
    -        """
    225 
    -
    226 
    -        self._name = str(uuid.uuid4())
    227 
    -        self._properties = {}
    228 
    -
    229 
    -        if properties:
    230 
    -            for k, v in properties.items():
    231 
    -                if k == 'os':
    232 
    -                    self._properties[k] = v
    233 
    -
    234 
    -                elif k == 'docker':
    235 
    -                    if v not in ('True', 'False'):
    236 
    -                        raise ValueError('Value not supported: [{}]'.format(v))
    237 
    -                    self._properties[k] = v
    238 
    -
    239 
    -                else:
    240 
    -                    raise KeyError('Key not supported: [{}]'.format(k))
    241 
    -
    242 
    -    @property
    243 
    -    def name(self):
    244 
    -        return self._name
    245 
    -
    246 
    -    @property
    247 
    -    def properties(self):
    248 
    -        return self._properties
    249 
    -
    250 
    -    def get_pb2(self):
    251 
    -        device = worker_pb2.Device(handle=self._name)
    252 
    -        property_message = worker_pb2.Device.Property()
    253 
    -        for k, v in self._properties.items():
    254 
    -            property_message.key = k
    255 
    -            property_message.value = v
    256 
    -            device.properties.extend([property_message])
    257 
    -        return device

  • buildgrid/bot/tenant.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +class LeaseState(Enum):
    17 
    +    # Default value.
    18 
    +    LEASE_STATE_UNSPECIFIED = bots_pb2.LeaseState.Value('LEASE_STATE_UNSPECIFIED')
    19 
    +
    20 
    +    # The server expects the bot to accept this lease.
    21 
    +    PENDING = bots_pb2.LeaseState.Value('PENDING')
    22 
    +
    23 
    +    # The bot has accepted this lease.
    24 
    +    ACTIVE = bots_pb2.LeaseState.Value('ACTIVE')
    25 
    +
    26 
    +    # The bot is no longer leased.
    27 
    +    COMPLETED = bots_pb2.LeaseState.Value('COMPLETED')
    28 
    +
    29 
    +    # The bot should immediately release all resources associated with the lease.
    30 
    +    CANCELLED = bots_pb2.LeaseState.Value('CANCELLED')
    31 
    +
    32 
    +
    33 
    +class Tenant:
    34 
    +
    35 
    +    def __init__(self, lease, context):
    36 
    +        self.logger = logging.getLogger(__name__)
    37 
    +        self._lease = lease
    38 
    +        self.__lease_cancelled = False
    39 
    +        self.__task = None
    40 
    +
    41 
    +    @property
    42 
    +    def state(self):
    43 
    +        return self._lease.state
    44 
    +
    45 
    +    def update_lease_state(self, state):
    46 
    +        self._lease.state = state
    47 
    +
    48 
    +    def cancel_lease(self):
    49 
    +        self.__lease_cancelled = True
    50 
    +        self.update_lease_state(LeaseState.CANCELLED)
    51 
    +
    52 
    +    async def run_work(self, lease):
    53 
    +        self.logger.debug("Work created: [{}]".format(lease.id))
    54 
    +        loop = asyncio.get_event_loop()
    55 
    +
    56 
    +        try:
    57 
    +            lease = await loop.run_in_executor(None, self._work, self._context, lease)
    58 
    +
    59 
    +        except grpc.RpcError as e:
    60 
    +            self.logger.error("RPC error thrown: [{}]".format(e))
    61 
    +            lease.status.CopyFrom(e.code())
    62 
    +
    63 
    +        except BotError as e:
    64 
    +            self.logger.error("Internal bot error thrown: [{}]".format(e))
    65 
    +            lease.status.code = code_pb2.INTERNAL
    66 
    +
    67 
    +        except Exception as e:
    68 
    +            self.logger.error("Exception thrown: [{}]".format(e))
    69 
    +            lease.status.code = code_pb2.INTERNAL
    70 
    +
    71 
    +        self.logger.debug("Work complete: [{}]".format(lease.id))
    72 
    +        self.lease_completed(lease)

  • buildgrid/bot/tenantmanager.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +"""
    17 
    +TenantManager
    18 
    +=============
    19 
    +
    20 
    +Looks after leases of work.
    21 
    +"""
    22 
    +
    23 
    +
    24 
    +import asyncio
    25 
    +
    26 
    +import grpc
    27 
    +
    28 
    +from .tenant import Tenant
    29 
    +
    30 
    +class TenantManager:
    31 
    +
    32 
    +    def __init__(self):
    33 
    +        self._tenants = {}
    34 
    +
    35 
    +    def get_lease_state(self, lease_id):
    36 
    +        return self._tenants[lease_id].state
    37 
    +
    38 
    +    def update_lease_state(self, lease_id, state):
    39 
    +        self._tenants[lease_id].update_lease_state(state)
    40 
    +
    41 
    +    def create_tenancy(self, lease, context):
    42 
    +        lease_id = lease.id
    43 
    +        if lease_id not in self._tenants:
    44 
    +            self._tenants[lease_id] = Tenant(lease, context)
    45 
    +
    46 
    +        raise Exception

  • buildgrid/bot/worker.py
    1 
    +# Copyright (C) 2018 Bloomberg LP
    2 
    +#
    3 
    +# Licensed under the Apache License, Version 2.0 (the "License");
    4 
    +# you may not use this file except in compliance with the License.
    5 
    +# You may obtain a copy of the License at
    6 
    +#
    7 
    +#  <http://www.apache.org/licenses/LICENSE-2.0>
    8 
    +#
    9 
    +# Unless required by applicable law or agreed to in writing, software
    10 
    +# distributed under the License is distributed on an "AS IS" BASIS,
    11 
    +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12 
    +# See the License for the specific language governing permissions and
    13 
    +# limitations under the License.
    14 
    +
    15 
    +
    16 
    +class Worker:
    17 
    +
    18 
    +    def __init__(self, properties=None, configs=None):
    19 
    +        self._devices = []
    20 
    +        self.__configs = {}
    21 
    +        self.__properties = {}
    22 
    +
    23 
    +        if properties:
    24 
    +            for k, v in properties.items():
    25 
    +                self.update_config(k, v)
    26 
    +
    27 
    +        if configs:
    28 
    +            for k, v in configs.items():
    29 
    +                if k == 'DockerImage':
    30 
    +                    self.__configs[k] = v
    31 
    +                else:
    32 
    +                    raise KeyError('Key not supported: [{}]'.format(k))
    33 
    +
    34 
    +    @property
    35 
    +    def configs(self):
    36 
    +        return self.__configs
    37 
    +
    38 
    +    @property
    39 
    +    def properties(self):
    40 
    +        return self.__properties
    41 
    +
    42 
    +    def update_config(self, key, value):
    43 
    +        """Update the config of the device.
    44 
    +        Only certain keys are allowed."""
    45 
    +        if k == 'pool':
    46 
    +            self.__properties[k] = v
    47 
    +        else:
    48 
    +            raise KeyError('Key not supported: [{}]'.format(k))
    49 
    +
    50 
    +    def add_device(self, device):
    51 
    +        self._devices.append(device)
    52 
    +
    53 
    +    def get_pb2(self):
    54 
    +        devices = [device.get_pb2() for device in self._devices]
    55 
    +        worker = worker_pb2.Worker(devices=devices)
    56 
    +        property_message = worker_pb2.Worker.Property()
    57 
    +        for k, v in self.__properties.items():
    58 
    +            property_message.key = k
    59 
    +            property_message.value = v
    60 
    +            worker.properties.extend([property_message])
    61 
    +
    62 
    +        config_message = worker_pb2.Worker.Config()
    63 
    +        for k, v in self.__properties.items():
    64 
    +            property_message.key = k
    65 
    +            property_message.value = v
    66 
    +            worker.configs.extend([config_message])
    67 
    +
    68 
    +        return worker

    • —
    View it on GitLab.
    You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.

    [Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]