[Notes] [Git][BuildGrid/buildgrid][finn/74-operation-cancelation] 5 commits: Add Device class.

[finn <gitlab mg gitlab com>]

Title: GitLab

finn pushed to branch finn/74-operation-cancelation at BuildGrid / buildgrid

Commits:

• ccc8604b
  by Finn at 2018-10-31T10:46:21Z

  Add Device class.
  

• c6a264e3
  by Finn at 2018-10-31T10:46:46Z

  Add Worker class.
  

• 238813a7
  by Finn at 2018-10-31T10:46:46Z

  Remove hardware descriptions from bot_session.
  
  Also move `bot_session` to simply `session`.
  

• d43f42b5
  by Finn at 2018-10-31T10:46:46Z

  Moved bots_interface.py to interface.py.
  

• bf5bb823
  by Finn at 2018-10-31T10:59:17Z

  Working commit
  

9 changed files:

• buildgrid/bot/bot.py
• + buildgrid/bot/hardware/__init__.py
• + buildgrid/bot/hardware/device.py
• + buildgrid/bot/hardware/interface.py
• + buildgrid/bot/hardware/worker.py
• buildgrid/bot/bot_interface.py → buildgrid/bot/interface.py
• buildgrid/bot/bot_session.py → buildgrid/bot/session.py
• + buildgrid/bot/tenant.py
• + buildgrid/bot/tenantmanager.py

Changes:

buildgrid/bot/bot.py

---

...	...	@@ -45,6 +45,7 @@ class Bot:
45	45	loop.run_forever()
46	46	except KeyboardInterrupt:
47	47	pass
	48	+
48	49	finally:
49	50	task.cancel()
50	51	loop.close()

buildgrid/bot/hardware/__init__.py

---

buildgrid/bot/hardware/device.py

---

	1	+# Copyright (C) 2018 Bloomberg LP
	2	+#
	3	+# Licensed under the Apache License, Version 2.0 (the "License");
	4	+# you may not use this file except in compliance with the License.
	5	+# You may obtain a copy of the License at
	6	+#
	7	+# <http://www.apache.org/licenses/LICENSE-2.0>
	8	+#
	9	+# Unless required by applicable law or agreed to in writing, software
	10	+# distributed under the License is distributed on an "AS IS" BASIS,
	11	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+# See the License for the specific language governing permissions and
	13	+# limitations under the License.
	14	+
	15	+
	16	+"""
	17	+Device
	18	+======
	19	+
	20	+A device.
	21	+"""
	22	+
	23	+class Device:
	24	+
	25	+ def __init__(self, properties=None):
	26	+ """ Creates devices available to the worker
	27	+ The first device is know as the Primary Device - the revice which
	28	+ is running a bit and responsible to actually executing commands.
	29	+ All other devices are known as Attatched Devices and must be controlled
	30	+ by the Primary Device.
	31	+
	32	+ properties (list(dict(string : string))) : Properties of device. Keys may
	33	+ repeated.
	34	+ """
	35	+
	36	+ self._properties = {}
	37	+ self.__property_keys = ['os', 'has-docker']
	38	+ self.__name = str(uuid.uuid4())
	39	+
	40	+ for prop in properties:
	41	+ self._add_property(prop)
	42	+
	43	+ @property
	44	+ def name(self):
	45	+ return self.__name
	46	+
	47	+ @property
	48	+ def properties(self):
	49	+ return self._properties
	50	+
	51	+ def get_pb2(self):
	52	+ device = worker_pb2.Device(handle=self._name)
	53	+ for k, v in self._properties.items():
	54	+ for prop in v:
	55	+ property_message = worker_pb2.Device.Property()
	56	+ property_message.key = k
	57	+ property_message.value = prop
	58	+ device.properties.extend([property_message])
	59	+ return device
	60	+
	61	+ def _add_property(self, key, value):
	62	+ if key in self.__property_keys:
	63	+ prop = self._properties.get(key)
	64	+ if not prop:
	65	+ self._properties[key] = [value]
	66	+ else:
	67	+ prop[key].append(value)
	68	+
	69	+ else:
	70	+ raise KeyError('Key not supported: [{}]'.format(key))

buildgrid/bot/hardware/interface.py

---

	1	+# Copyright (C) 2018 Bloomberg LP
	2	+#
	3	+# Licensed under the Apache License, Version 2.0 (the "License");
	4	+# you may not use this file except in compliance with the License.
	5	+# You may obtain a copy of the License at
	6	+#
	7	+# <http://www.apache.org/licenses/LICENSE-2.0>
	8	+#
	9	+# Unless required by applicable law or agreed to in writing, software
	10	+# distributed under the License is distributed on an "AS IS" BASIS,
	11	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+# See the License for the specific language governing permissions and
	13	+# limitations under the License.
	14	+
	15	+
	16	+"""
	17	+HardwareInterface
	18	+=================
	19	+
	20	+Class to configure hardware and check requirements of leases.
	21	+"""
	22	+
	23	+from buildgrid._exceptions import FailedPreconditionError
	24	+
	25	+
	26	+class HardwareInterface:
	27	+
	28	+ def __init__(self, worker):
	29	+ self._worker = worker
	30	+
	31	+ def configure_hardware(self, lease):
	32	+ """ Can check if the requirements can be met and also
	33	+ in the future, potentially configure the hardware.
	34	+ """
	35	+ worker = self._worker
	36	+ worker_requirements = lease.worker
	37	+
	38	+ for config_requirement in worker_requirements.configs:
	39	+ if config_requirement.key not in worker.configs:
	40	+ raise FailedPreconditionError("Config not supported: [{}]".format(config_requirement))
	41	+
	42	+ def get_worker_pb2(self):
	43	+ return self._worker.get_pb2()

buildgrid/bot/hardware/worker.py

---

	1	+# Copyright (C) 2018 Bloomberg LP
	2	+#
	3	+# Licensed under the Apache License, Version 2.0 (the "License");
	4	+# you may not use this file except in compliance with the License.
	5	+# You may obtain a copy of the License at
	6	+#
	7	+# <http://www.apache.org/licenses/LICENSE-2.0>
	8	+#
	9	+# Unless required by applicable law or agreed to in writing, software
	10	+# distributed under the License is distributed on an "AS IS" BASIS,
	11	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+# See the License for the specific language governing permissions and
	13	+# limitations under the License.
	14	+
	15	+
	16	+class Worker:
	17	+
	18	+ def __init__(self, properties=None, configs=None):
	19	+ self._devices = []
	20	+ self._configs = {}
	21	+ self._properties = {}
	22	+ self.__property_keys = ['pool']
	23	+ self.__config_keys = ['DockerImage']
	24	+
	25	+ if properties:
	26	+ for k, v in properties.items():
	27	+ if k in self.__property_keys:
	28	+ self._add_properties(k, v)
	29	+
	30	+ if configs:
	31	+ for k, v in configs.items():
	32	+ self._add_config(k, v)
	33	+
	34	+ @property
	35	+ def configs(self):
	36	+ return self._configs
	37	+
	38	+ @property
	39	+ def properties(self):
	40	+ return self._properties
	41	+
	42	+ def add_device(self, device):
	43	+ self._devices.append(device)
	44	+
	45	+ def get_pb2(self):
	46	+ devices = [device.get_pb2() for device in self._devices]
	47	+ worker = worker_pb2.Worker(devices=devices)
	48	+
	49	+ for k, v in self._properties.items():
	50	+ for prop in v:
	51	+ property_message = worker_pb2.Device.Property()
	52	+ property_message.key = k
	53	+ property_message.value = prop
	54	+ device.properties.extend([property_message])
	55	+
	56	+ for k, v in self._configs.items():
	57	+ for cfg in v:
	58	+ config_message = worker_pb2.Worker.Config()
	59	+ config.key = k
	60	+ config_message.value = cfg
	61	+ worker.configs.extend([config_message])
	62	+
	63	+ return worker
	64	+
	65	+ def _add_config(self, key, value):
	66	+ if key in self.__config_keys:
	67	+ cfg = self._configs.get(key)
	68	+ if not cfg:
	69	+ self._configs[key] = [value]
	70	+ else:
	71	+ cfg[key].append(value)
	72	+
	73	+ else:
	74	+ raise KeyError('Key not supported: [{}]'.format(key))
	75	+
	76	+ def _add_property(self, key, value):
	77	+ if key in self.__property_keys:
	78	+ prop = self._properties.get(key)
	79	+ if not prop:
	80	+ self._properties[key] = [value]
	81	+ else:
	82	+ prop[key].append(value)
	83	+
	84	+ else:
	85	+ raise KeyError('Key not supported: [{}]'.format(key))

buildgrid/bot/bot_interface.py → buildgrid/bot/interface.py

---

...	...	@@ -15,7 +15,7 @@
15	15
16	16	"""
17	17	Bot Interface
18		-====
	18	+=============
19	19
20	20	Interface to grpc
21	21	"""

buildgrid/bot/bot_session.py → buildgrid/bot/session.py

---

...	...	@@ -18,7 +18,7 @@
18	18
19	19	"""
20	20	Bot Session
21		-====
	21	+===========
22	22
23	23	Allows connections
24	24	"""
...	...	@@ -36,7 +36,7 @@ from buildgrid._exceptions import BotError
36	36
37	37
38	38	class BotSession:
39		- def __init__(self, parent, interface):
	39	+ def __init__(self, parent, interface, hardware):
40	40	""" Unique bot ID within the farm used to identify this bot
41	41	Needs to be human readable.
42	42	All prior sessions with bot_id of same ID are invalidated.
...	...	@@ -45,39 +45,34 @@ class BotSession:
45	45	"""
46	46
47	47	self.logger = logging.getLogger(__name__)
	48	+ self._tenant_manager = TenantManager()
48	49
49		- self._bot_id = '{}.{}'.format(parent, platform.node())
50		- self._context = None
51	50	self._interface = interface
52		- self._leases = {}
53		- self._name = None
54		- self._parent = parent
	51	+ self._hardware = hardware
55	52	self._status = BotStatus.OK.value
56		- self._work = None
57		- self._worker = None
	53	+
	54	+ self.__parent = parent
	55	+ self.__bot_id = '{}.{}'.format(parent, platform.node())
	56	+ self.__name = None
58	57
59	58	@property
60	59	def bot_id(self):
61		- return self._bot_id
62		-
63		- def add_worker(self, worker):
64		- self._worker = worker
	60	+ return self.__bot_id
65	61
66		- def create_bot_session(self, work, context=None):
	62	+ def create_bot_session(self):
67	63	self.logger.debug("Creating bot session")
68		- self._work = work
69		- self._context = context
70	64
71	65	session = self._interface.create_bot_session(self._parent, self.get_pb2())
72		- self._name = session.name
	66	+ self.__name = session.name
73	67
74		- self.logger.info("Created bot session with name: [{}]".format(self._name))
	68	+ self.logger.info("Created bot session with name: [{}]".format(self.__name))
75	69
76	70	for lease in session.leases:
77		- self._update_lease_from_server(lease)
	71	+ if self._check_requirements(lease):
	72	+ self._tenant_manager.create_tenancy(lease)
78	73
79	74	def update_bot_session(self):
80		- self.logger.debug("Updating bot session: [{}]".format(self._bot_id))
	75	+ self.logger.debug("Updating bot session: [{}]".format(self.__bot_id))
81	76	session = self._interface.update_bot_session(self.get_pb2())
82	77	for k, v in list(self._leases.items()):
83	78	if v.state == LeaseState.COMPLETED.value:
...	...	@@ -94,130 +89,5 @@ class BotSession:
94	89	return bots_pb2.BotSession(worker=self._worker.get_pb2(),
95	90	status=self._status,
96	91	leases=leases,
97		- bot_id=self._bot_id,
98		- name=self._name)
99		-
100		- def lease_completed(self, lease):
101		- lease.state = LeaseState.COMPLETED.value
102		- self._leases[lease.id] = lease
103		-
104		- def _update_lease_from_server(self, lease):
105		- """
106		- State machine for any recieved updates to the leases.
107		- """
108		- # TODO: Compare with previous state of lease
109		- if lease.state == LeaseState.PENDING.value:
110		- lease.state = LeaseState.ACTIVE.value
111		- self._leases[lease.id] = lease
112		- self.update_bot_session()
113		- asyncio.ensure_future(self.create_work(lease))
114		-
115		- async def create_work(self, lease):
116		- self.logger.debug("Work created: [{}]".format(lease.id))
117		- loop = asyncio.get_event_loop()
118		-
119		- try:
120		- lease = await loop.run_in_executor(None, self._work, self._context, lease)
121		-
122		- except grpc.RpcError as e:
123		- self.logger.error("RPC error thrown: [{}]".format(e))
124		- lease.status.CopyFrom(e.code())
125		-
126		- except BotError as e:
127		- self.logger.error("Internal bot error thrown: [{}]".format(e))
128		- lease.status.code = code_pb2.INTERNAL
129		-
130		- except Exception as e:
131		- self.logger.error("Exception thrown: [{}]".format(e))
132		- lease.status.code = code_pb2.INTERNAL
133		-
134		- self.logger.debug("Work complete: [{}]".format(lease.id))
135		- self.lease_completed(lease)
136		-
137		-
138		-class Worker:
139		- def __init__(self, properties=None, configs=None):
140		- self.properties = {}
141		- self._configs = {}
142		- self._devices = []
143		-
144		- if properties:
145		- for k, v in properties.items():
146		- if k == 'pool':
147		- self.properties[k] = v
148		- else:
149		- raise KeyError('Key not supported: [{}]'.format(k))
150		-
151		- if configs:
152		- for k, v in configs.items():
153		- if k == 'DockerImage':
154		- self.configs[k] = v
155		- else:
156		- raise KeyError('Key not supported: [{}]'.format(k))
157		-
158		- @property
159		- def configs(self):
160		- return self._configs
161		-
162		- def add_device(self, device):
163		- self._devices.append(device)
164		-
165		- def get_pb2(self):
166		- devices = [device.get_pb2() for device in self._devices]
167		- worker = worker_pb2.Worker(devices=devices)
168		- property_message = worker_pb2.Worker.Property()
169		- for k, v in self.properties.items():
170		- property_message.key = k
171		- property_message.value = v
172		- worker.properties.extend([property_message])
173		-
174		- config_message = worker_pb2.Worker.Config()
175		- for k, v in self.properties.items():
176		- property_message.key = k
177		- property_message.value = v
178		- worker.configs.extend([config_message])
179		-
180		- return worker
181		-
182		-
183		-class Device:
184		- def __init__(self, properties=None):
185		- """ Creates devices available to the worker
186		- The first device is know as the Primary Device - the revice which
187		- is running a bit and responsible to actually executing commands.
188		- All other devices are known as Attatched Devices and must be controlled
189		- by the Primary Device.
190		- """
191		-
192		- self._name = str(uuid.uuid4())
193		- self._properties = {}
194		-
195		- if properties:
196		- for k, v in properties.items():
197		- if k == 'os':
198		- self._properties[k] = v
199		-
200		- elif k == 'docker':
201		- if v not in ('True', 'False'):
202		- raise ValueError('Value not supported: [{}]'.format(v))
203		- self._properties[k] = v
204		-
205		- else:
206		- raise KeyError('Key not supported: [{}]'.format(k))
207		-
208		- @property
209		- def name(self):
210		- return self._name
211		-
212		- @property
213		- def properties(self):
214		- return self._properties
215		-
216		- def get_pb2(self):
217		- device = worker_pb2.Device(handle=self._name)
218		- property_message = worker_pb2.Device.Property()
219		- for k, v in self._properties.items():
220		- property_message.key = k
221		- property_message.value = v
222		- device.properties.extend([property_message])
223		- return device
	92	+ bot_id=self.__bot_id,
	93	+ name=self.__name)

buildgrid/bot/tenant.py

---

	1	+# Copyright (C) 2018 Bloomberg LP
	2	+#
	3	+# Licensed under the Apache License, Version 2.0 (the "License");
	4	+# you may not use this file except in compliance with the License.
	5	+# You may obtain a copy of the License at
	6	+#
	7	+# <http://www.apache.org/licenses/LICENSE-2.0>
	8	+#
	9	+# Unless required by applicable law or agreed to in writing, software
	10	+# distributed under the License is distributed on an "AS IS" BASIS,
	11	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+# See the License for the specific language governing permissions and
	13	+# limitations under the License.
	14	+
	15	+"""
	16	+Tenant
	17	+======
	18	+
	19	+Handles leased and runs leased work.
	20	+"""
	21	+
	22	+
	23	+from functools import partial
	24	+
	25	+from buildgrid._enums import LeaseState
	26	+
	27	+
	28	+class Tenant:
	29	+
	30	+ def __init__(self, lease, context):
	31	+
	32	+ if lease.state != LeaseState.PENDING:
	33	+ raise ValueError("Lease state not `PENDING`: {}".format(lease.state))
	34	+
	35	+ self.logger = logging.getLogger(__name__)
	36	+ self._context = context
	37	+ self._lease = lease
	38	+ self.__lease_cancelled = False
	39	+
	40	+ def get_state_state(self):
	41	+ return self._lease.state
	42	+
	43	+ def update_lease_state(self, state):
	44	+ self._lease.state = state
	45	+
	46	+ def cancel_lease(self):
	47	+ self.__lease_cancelled = True
	48	+ self.update_lease_state(LeaseState.CANCELLED)
	49	+
	50	+ async def run_work(self, work, executor=None, context = None):
	51	+ self.logger.debug("Work created: [{}]".format(lease.id))
	52	+ loop = asyncio.get_event_loop()
	53	+
	54	+ try:
	55	+ lease = await loop.run_in_executor(executor, partial(work, context, self._lease))
	56	+
	57	+ except asyncio.CancelledError as e:
	58	+ self.logger.error("Task cancelled: [{}]".format(e))
	59	+
	60	+ except grpc.RpcError as e:
	61	+ self.logger.error("RPC error thrown: [{}]".format(e))
	62	+ lease.status.CopyFrom(e.code())
	63	+
	64	+ except BotError as e:
	65	+ self.logger.error("Internal bot error thrown: [{}]".format(e))
	66	+ lease.status.code = code_pb2.INTERNAL
	67	+
	68	+ except Exception as e:
	69	+ self.logger.error("Exception thrown: [{}]".format(e))
	70	+ lease.status.code = code_pb2.INTERNAL
	71	+
	72	+ self.logger.debug("Work complete: [{}]".format(lease.id))

buildgrid/bot/tenantmanager.py

---

	1	+# Copyright (C) 2018 Bloomberg LP
	2	+#
	3	+# Licensed under the Apache License, Version 2.0 (the "License");
	4	+# you may not use this file except in compliance with the License.
	5	+# You may obtain a copy of the License at
	6	+#
	7	+# <http://www.apache.org/licenses/LICENSE-2.0>
	8	+#
	9	+# Unless required by applicable law or agreed to in writing, software
	10	+# distributed under the License is distributed on an "AS IS" BASIS,
	11	+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+# See the License for the specific language governing permissions and
	13	+# limitations under the License.
	14	+
	15	+
	16	+"""
	17	+TenantManager
	18	+=============
	19	+
	20	+Looks after leases of work.
	21	+"""
	22	+
	23	+
	24	+import asyncio
	25	+from functools import partial
	26	+
	27	+import grpc
	28	+
	29	+from buildgrid._enums import LeaseState
	30	+
	31	+from .tenant import Tenant
	32	+
	33	+class TenantManager:
	34	+
	35	+ def __init__(self):
	36	+ self._tenants = {}
	37	+
	38	+ def get_lease_state(self, lease_id):
	39	+ return self._tenants[lease_id].get_lease_state()
	40	+
	41	+ def update_lease_state(self, lease_id, state):
	42	+ self._tenants[lease_id].update_lease_state(state)
	43	+
	44	+ def create_tenancy(self, lease, context):
	45	+ lease_id = lease.id
	46	+
	47	+ if lease_id not in self._tenants:
	48	+ tenant = Tenant(lease, context)
	49	+ tenant.update_lease_state(LeaseState.ACTIVE)
	50	+ self._tenants[lease_id] = tenant
	51	+ task = asyncio.ensure_future(self.create_work(lease))
	52	+ task.add_done_callback(partial(self.update_lease_state(lease_id, LeaseState.COMPLETED)))
	53	+
	54	+ raise KeyError("Lease id already exists: [{}]".format(lease_id))
	55	+
	56	+ def cancel_tenancy(self, lease_id):
	57	+ self._tenants[lease_id].cancel_lease()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.