[Notes] [Git][BuildGrid/buildgrid][master] 2 commits: Worker side now uses scheduler to monitor leases.

[Agustin Benito Bethencourt <gitlab mg gitlab com>]

Title: GitLab

Agustin Benito Bethencourt pushed to branch master at BuildGrid / buildgrid

Commits:

• e3333b9d
  by finn at 2018-07-24T16:19:50Z

  Worker side now uses scheduler to monitor leases.
  

• 71ffff5a
  by Carter Sande at 2018-07-24T19:15:34Z

  Add google.rpc.code protobuf library
  

10 changed files:

• buildgrid/server/job.py
• buildgrid/server/scheduler.py
• buildgrid/server/worker/bots_interface.py
• + google/rpc/code.proto
• + google/rpc/code_pb2.py
• + google/rpc/code_pb2_grpc.py
• + tests/integration/__init__.py
• tests/integration/bots_service.py
• tests/job.py
• tests/scheduler.py

Changes:

buildgrid/server/job.py

---

...	...	@@ -55,7 +55,7 @@ class Job():
55	55	self.action = action
56	56	self.bot_status = BotStatus.BOT_STATUS_UNSPECIFIED
57	57	self.execute_stage = ExecuteStage.UNKNOWN
58		- self.lease_state = LeaseState.LEASE_STATE_UNSPECIFIED
	58	+ self.lease = None
59	59	self.logger = logging.getLogger(__name__)
60	60	self.name = str(uuid.uuid4())
61	61	self.result = None
...	...	@@ -78,18 +78,15 @@ class Job():
78	78
79	79	return meta
80	80
81		- def get_lease(self):
	81	+ def create_lease(self):
82	82	action = self._pack_any(self.action)
83	83
84	84	lease = bots_pb2.Lease(assignment = self.name,
85	85	inline_assignment = action,
86		- state = self.lease_state.value)
	86	+ state = LeaseState.PENDING.value)
	87	+ self.lease = lease
87	88	return lease
88	89
89		- def cancel(self):
90		- # TODO: Handle cancelled jobs
91		- pass
92		-
93	90	def get_operations(self):
94	91	return operations_pb2.ListOperationsResponse(operations = [self.get_operation()])
95	92

buildgrid/server/scheduler.py

---

...	...	@@ -44,8 +44,8 @@ class Scheduler():
44	44	job = self.jobs[name]
45	45
46	46	if job.n_tries >= self.MAX_N_TRIES:
	47	+ # TODO: Decide what to do with these jobs
47	48	job.execute_stage = ExecuteStage.COMPLETED
48		- job.cancel()
49	49	else:
50	50	job.execute_stage = ExecuteStage.QUEUED
51	51	job.n_tries += 1
...	...	@@ -53,12 +53,13 @@ class Scheduler():
53	53
54	54	self.jobs[name] = job
55	55
56		- def get_job(self):
57		- job = self.queue.popleft()
58		- job.execute_stage = ExecuteStage.EXECUTING
59		- job.lease_state = LeaseState.PENDING
60		- self.jobs[job.name] = job
61		- return job
	56	+ def create_job(self):
	57	+ if len(self.queue) > 0:
	58	+ job = self.queue.popleft()
	59	+ job.execute_stage = ExecuteStage.EXECUTING
	60	+ self.jobs[job.name] = job
	61	+ return job
	62	+ return None
62	63
63	64	def job_complete(self, name, result):
64	65	job = self.jobs[name]
...	...	@@ -71,3 +72,53 @@ class Scheduler():
71	72	for v in self.jobs.values():
72	73	response.operations.extend([v.get_operation()])
73	74	return response
	75	+
	76	+ def update_lease(self, lease):
	77	+ name = lease.assignment
	78	+ job = self.jobs.get(name)
	79	+ state = lease.state
	80	+
	81	+ if state == LeaseState.LEASE_STATE_UNSPECIFIED.value:
	82	+ create_job = self.create_job()
	83	+ if create_job is None:
	84	+ # No job? Return lease.
	85	+ return lease
	86	+ else:
	87	+ job = create_job
	88	+ job.lease = job.create_lease()
	89	+
	90	+ elif state == LeaseState.PENDING.value:
	91	+ job.lease = lease
	92	+
	93	+ elif state == LeaseState.ACTIVE.value:
	94	+ job.lease = lease
	95	+
	96	+ elif state == LeaseState.COMPLETED.value:
	97	+ self.job_complete(job.name, lease.inline_assignment)
	98	+
	99	+ create_job = self.create_job()
	100	+ if create_job is None:
	101	+ # Docs say not to use this state though if job has
	102	+ # completed and no more jobs, then use this state to stop
	103	+ # job being processed again
	104	+ job.lease = lease
	105	+ job.lease.state = LeaseState.LEASE_STATE_UNSPECIFIED.value
	106	+ else:
	107	+ job = create_job
	108	+ job.lease = job.create_lease()
	109	+
	110	+ elif state == LeaseState.CANCELLED.value:
	111	+ job.lease = lease
	112	+
	113	+ else:
	114	+ raise Exception("Unknown state: {}".format(state))
	115	+
	116	+ self.jobs[name] = job
	117	+ return job.lease
	118	+
	119	+ def cancel_session(self, name):
	120	+ job = self.jobs[name]
	121	+ state = job.lease.state
	122	+ if state == LeaseState.PENDING.value or \
	123	+ state == LeaseState.ACTIVE.value:
	124	+ self.retry_job(name)

buildgrid/server/worker/bots_interface.py

---

...	...	@@ -34,7 +34,7 @@ class BotsInterface():
34	34	def __init__(self, scheduler):
35	35	self.logger = logging.getLogger(__name__)
36	36
37		- self._bots = {}
	37	+ self._bot_ids = {}
38	38	self._scheduler = scheduler
39	39
40	40	def create_bot_session(self, parent, bot_session):
...	...	@@ -49,13 +49,16 @@ class BotsInterface():
49	49	if bot_id == "":
50	50	raise InvalidArgumentError("bot_id needs to be set by client")
51	51
52		- self._check_bot_ids(bot_id)
	52	+ try:
	53	+ self._check_bot_ids(bot_id)
	54	+ except InvalidArgumentError:
	55	+ pass
53	56
54	57	# Bot session name, selected by the server
55	58	name = str(uuid.uuid4())
56	59	bot_session.name = name
57	60
58		- self._bots[name] = bot_session
	61	+ self._bot_ids[name] = bot_id
59	62	self.logger.info("Created bot session name={} with bot_id={}".format(name, bot_id))
60	63	return bot_session
61	64
...	...	@@ -64,115 +67,43 @@ class BotsInterface():
64	67	registered server side. Assigns available leases with work.
65	68	"""
66	69	self.logger.debug("Updating bot session name={}".format(name))
67		- bot = self._bots.get(name)
68		-
69		- if bot == None:
70		- self.logger.warn("Update received for {} but not found on server ({}).".format(name,
71		- bot_session.bot_id))
72		- raise InvalidArgumentError("Bot with name={} is not registered on server.".format(name))
73		- else:
74		- leases_server = bot.leases
75		-
76		- # Close any zombies
77		- if not self._check_bot_ids(name, bot_session):
78		- raise InvalidArgumentError("Bot with name={} has incorrect bot_id={}.".format(name,
79		- bot_session.bot_id))
80		-
81		- leases_client = bot_session.leases
82		- if len(leases_client) != len(leases_server):
83		- self._close_bot_session(name)
84		- raise OutofSyncError("Number of leases in server and client not same."+\
85		- "Closed bot session: {}".format(name)+\
86		- "Client: {}\nServer: {}".format(len(leases_client), len(leases_server)))
	70	+ self._check_bot_ids(bot_session.bot_id, name)
87	71
88		- leases_client = [self._check_lease(lease) for lease in leases_client]
	72	+ leases = [self._scheduler.update_lease(lease) for lease in bot_session.leases]
89	73
90	74	del bot_session.leases[:]
91		- bot_session.leases.extend(leases_client)
	75	+ bot_session.leases.extend(leases)
92	76
93		- self._bots[name] = bot_session
94	77	return bot_session
95	78
96	79	def _check_bot_ids(self, bot_id, name = None):
97		- ''' Generate a list of all the bots that are reporting with this id but
98		- not this name. Per the spec, any bot that is reporting an ID that
99		- does not match the name we have file for them should not be given any
100		- work. Returns False if the check fails and a bot_session is closed.'''
101		-
102		- for _name, bot in list(self._bots.items()):
103		- if bot.bot_id == bot_id and _name != name:
104		- self.logger.warn("Duplicate bot_id provided of {}: this is registered under names {} and {}. Closing session with {}."
105		- .format(bot_id, name, _name, _name))
106		- self._close_bot_session(_name)
107		- return False
108		- return True
109		-
110		- def _check_lease(self, lease):
111		- """ Checks status of lease
	80	+ """ Checks the ID and the name of the bot.
112	81	"""
113		- state = lease.state
114		-
115		- if state == LeaseState.LEASE_STATE_UNSPECIFIED.value:
116		- return self._get_pending_action(lease)
117		-
118		- elif state == LeaseState.PENDING.value:
119		- return lease
120		-
121		- elif state == LeaseState.ACTIVE.value:
122		- return lease
123		-
124		- elif state == LeaseState.COMPLETED.value:
125		- name = lease.assignment
126		- result = lease.inline_assignment
127		- self._scheduler.job_complete(name, result)
	82	+ if name is not None:
	83	+ _bot_id = self._bot_ids.get(name)
	84	+ if _bot_id is None:
	85	+ raise InvalidArgumentError('Name not registered on server: {}'.format(name))
	86	+ elif _bot_id != bot_id:
	87	+ self._close_bot_session(name)
	88	+ raise InvalidArgumentError('Bot id invalid. ID sent: {} with name: {}. ID registered: {} with name: {}'.format(bot_id, name, _bot_id, _name))
128	89
129		- return self._get_pending_action(lease)
130		-
131		- elif state == LeaseState.CANCELLED.value:
132		- # TODO: Add cancelled message to result
133		- raise NotImplementedError
134		-
135		- else:
136		- raise InvalidArgumentError("Unknown state: {}".format(state))
137		-
138		- def _get_pending_action(self, lease):
139		- """ If actions are available, populates the lease and
140		- informs the execution service, else it returns the lease.
141		- """
142		- if len(self._scheduler.queue) > 0:
143		- job = self._scheduler.get_job()
144		- lease = job.get_lease()
145	90	else:
146		- # Doc says not to use this sate.
147		- # Though if lease has completed, no need to process again
148		- # if there are no pending actions
149		- lease.state = LeaseState.LEASE_STATE_UNSPECIFIED.value
150		-
151		- return lease
152		-
153		- def _requeue_lease_if_applicable(self, lease):
154		- state = lease.state
155		- if state == LeaseState.PENDING.value or \
156		- state == LeaseState.ACTIVE.value:
157		- name = lease.assignment
158		- action = lease.inline_assignment
159		- self._scheduler.jobs[name].retry_job(name)
	91	+ for _name, _bot_id in self._bot_ids.items():
	92	+ if bot_id == _bot_id:
	93	+ self._close_bot_session(_name)
	94	+ raise InvalidArgumentError('Bot id already registered. ID sent: {}. Id registered: {} with name: {}'.format(bot_id, _bot_id, _name))
160	95
161	96	def _close_bot_session(self, name):
162	97	""" Before removing the session, close any leases and
163	98	requeue with high priority.
164	99	"""
165		- bot = self._bots.get(name)
	100	+ bot_id = self._bot_ids.get(name)
166	101
167		- if bot is None:
168		- raise InvalidArgumentError("Bot name does not exist: {}".format(name))
	102	+ if bot_id is None:
	103	+ raise InvalidArgumentError("Bot id does not exist: {}".format(name))
169	104
170		- self.logger.debug("Attempting to close {} with name: {}".format(bot.bot_id, name))
171		- try:
172		- for lease in bot.leases:
173		- self._requeue_lease_if_applicable(lease)
174		- self.logger.debug("Closing bot session: {}".format(name))
175		- self._bots.pop(name)
176		- self.logger.info("Closed bot {} with name: {}".format(bot.bot_id, name))
177		- except KeyError:
178		- raise InvalidArgumentError("Bot name does not exist: {}".format(name))
	105	+ self.logger.debug("Attempting to close {} with name: {}".format(bot_id, name))
	106	+ self._scheduler.retry_job(name)
	107	+ self.logger.debug("Closing bot session: {}".format(name))
	108	+ self._bot_ids.pop(name)
	109	+ self.logger.info("Closed bot {} with name: {}".format(bot_id, name))

google/rpc/code.proto

---

	1	+// Copyright 2017 Google Inc.
	2	+//
	3	+// Licensed under the Apache License, Version 2.0 (the "License");
	4	+// you may not use this file except in compliance with the License.
	5	+// You may obtain a copy of the License at
	6	+//
	7	+// http://www.apache.org/licenses/LICENSE-2.0
	8	+//
	9	+// Unless required by applicable law or agreed to in writing, software
	10	+// distributed under the License is distributed on an "AS IS" BASIS,
	11	+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	12	+// See the License for the specific language governing permissions and
	13	+// limitations under the License.
	14	+
	15	+syntax = "proto3";
	16	+
	17	+package google.rpc;
	18	+
	19	+option go_package = "google.golang.org/genproto/googleapis/rpc/code;code";
	20	+option java_multiple_files = true;
	21	+option java_outer_classname = "CodeProto";
	22	+option java_package = "com.google.rpc";
	23	+option objc_class_prefix = "RPC";
	24	+
	25	+
	26	+// The canonical error codes for Google APIs.
	27	+//
	28	+//
	29	+// Sometimes multiple error codes may apply. Services should return
	30	+// the most specific error code that applies. For example, prefer
	31	+// `OUT_OF_RANGE` over `FAILED_PRECONDITION` if both codes apply.
	32	+// Similarly prefer `NOT_FOUND` or `ALREADY_EXISTS` over `FAILED_PRECONDITION`.
	33	+enum Code {
	34	+ // Not an error; returned on success
	35	+ //
	36	+ // HTTP Mapping: 200 OK
	37	+ OK = 0;
	38	+
	39	+ // The operation was cancelled, typically by the caller.
	40	+ //
	41	+ // HTTP Mapping: 499 Client Closed Request
	42	+ CANCELLED = 1;
	43	+
	44	+ // Unknown error. For example, this error may be returned when
	45	+ // a `Status` value received from another address space belongs to
	46	+ // an error space that is not known in this address space. Also
	47	+ // errors raised by APIs that do not return enough error information
	48	+ // may be converted to this error.
	49	+ //
	50	+ // HTTP Mapping: 500 Internal Server Error
	51	+ UNKNOWN = 2;
	52	+
	53	+ // The client specified an invalid argument. Note that this differs
	54	+ // from `FAILED_PRECONDITION`. `INVALID_ARGUMENT` indicates arguments
	55	+ // that are problematic regardless of the state of the system
	56	+ // (e.g., a malformed file name).
	57	+ //
	58	+ // HTTP Mapping: 400 Bad Request
	59	+ INVALID_ARGUMENT = 3;
	60	+
	61	+ // The deadline expired before the operation could complete. For operations
	62	+ // that change the state of the system, this error may be returned
	63	+ // even if the operation has completed successfully. For example, a
	64	+ // successful response from a server could have been delayed long
	65	+ // enough for the deadline to expire.
	66	+ //
	67	+ // HTTP Mapping: 504 Gateway Timeout
	68	+ DEADLINE_EXCEEDED = 4;
	69	+
	70	+ // Some requested entity (e.g., file or directory) was not found.
	71	+ //
	72	+ // Note to server developers: if a request is denied for an entire class
	73	+ // of users, such as gradual feature rollout or undocumented whitelist,
	74	+ // `NOT_FOUND` may be used. If a request is denied for some users within
	75	+ // a class of users, such as user-based access control, `PERMISSION_DENIED`
	76	+ // must be used.
	77	+ //
	78	+ // HTTP Mapping: 404 Not Found
	79	+ NOT_FOUND = 5;
	80	+
	81	+ // The entity that a client attempted to create (e.g., file or directory)
	82	+ // already exists.
	83	+ //
	84	+ // HTTP Mapping: 409 Conflict
	85	+ ALREADY_EXISTS = 6;
	86	+
	87	+ // The caller does not have permission to execute the specified
	88	+ // operation. `PERMISSION_DENIED` must not be used for rejections
	89	+ // caused by exhausting some resource (use `RESOURCE_EXHAUSTED`
	90	+ // instead for those errors). `PERMISSION_DENIED` must not be
	91	+ // used if the caller can not be identified (use `UNAUTHENTICATED`
	92	+ // instead for those errors). This error code does not imply the
	93	+ // request is valid or the requested entity exists or satisfies
	94	+ // other pre-conditions.
	95	+ //
	96	+ // HTTP Mapping: 403 Forbidden
	97	+ PERMISSION_DENIED = 7;
	98	+
	99	+ // The request does not have valid authentication credentials for the
	100	+ // operation.
	101	+ //
	102	+ // HTTP Mapping: 401 Unauthorized
	103	+ UNAUTHENTICATED = 16;
	104	+
	105	+ // Some resource has been exhausted, perhaps a per-user quota, or
	106	+ // perhaps the entire file system is out of space.
	107	+ //
	108	+ // HTTP Mapping: 429 Too Many Requests
	109	+ RESOURCE_EXHAUSTED = 8;
	110	+
	111	+ // The operation was rejected because the system is not in a state
	112	+ // required for the operation's execution. For example, the directory
	113	+ // to be deleted is non-empty, an rmdir operation is applied to
	114	+ // a non-directory, etc.
	115	+ //
	116	+ // Service implementors can use the following guidelines to decide
	117	+ // between `FAILED_PRECONDITION`, `ABORTED`, and `UNAVAILABLE`:
	118	+ // (a) Use `UNAVAILABLE` if the client can retry just the failing call.
	119	+ // (b) Use `ABORTED` if the client should retry at a higher level
	120	+ // (e.g., when a client-specified test-and-set fails, indicating the
	121	+ // client should restart a read-modify-write sequence).
	122	+ // (c) Use `FAILED_PRECONDITION` if the client should not retry until
	123	+ // the system state has been explicitly fixed. E.g., if an "rmdir"
	124	+ // fails because the directory is non-empty, `FAILED_PRECONDITION`
	125	+ // should be returned since the client should not retry unless
	126	+ // the files are deleted from the directory.
	127	+ //
	128	+ // HTTP Mapping: 400 Bad Request
	129	+ FAILED_PRECONDITION = 9;
	130	+
	131	+ // The operation was aborted, typically due to a concurrency issue such as
	132	+ // a sequencer check failure or transaction abort.
	133	+ //
	134	+ // See the guidelines above for deciding between `FAILED_PRECONDITION`,
	135	+ // `ABORTED`, and `UNAVAILABLE`.
	136	+ //
	137	+ // HTTP Mapping: 409 Conflict
	138	+ ABORTED = 10;
	139	+
	140	+ // The operation was attempted past the valid range. E.g., seeking or
	141	+ // reading past end-of-file.
	142	+ //
	143	+ // Unlike `INVALID_ARGUMENT`, this error indicates a problem that may
	144	+ // be fixed if the system state changes. For example, a 32-bit file
	145	+ // system will generate `INVALID_ARGUMENT` if asked to read at an
	146	+ // offset that is not in the range [0,2^32-1], but it will generate
	147	+ // `OUT_OF_RANGE` if asked to read from an offset past the current
	148	+ // file size.
	149	+ //
	150	+ // There is a fair bit of overlap between `FAILED_PRECONDITION` and
	151	+ // `OUT_OF_RANGE`. We recommend using `OUT_OF_RANGE` (the more specific
	152	+ // error) when it applies so that callers who are iterating through
	153	+ // a space can easily look for an `OUT_OF_RANGE` error to detect when
	154	+ // they are done.
	155	+ //
	156	+ // HTTP Mapping: 400 Bad Request
	157	+ OUT_OF_RANGE = 11;
	158	+
	159	+ // The operation is not implemented or is not supported/enabled in this
	160	+ // service.
	161	+ //
	162	+ // HTTP Mapping: 501 Not Implemented
	163	+ UNIMPLEMENTED = 12;
	164	+
	165	+ // Internal errors. This means that some invariants expected by the
	166	+ // underlying system have been broken. This error code is reserved
	167	+ // for serious errors.
	168	+ //
	169	+ // HTTP Mapping: 500 Internal Server Error
	170	+ INTERNAL = 13;
	171	+
	172	+ // The service is currently unavailable. This is most likely a
	173	+ // transient condition, which can be corrected by retrying with
	174	+ // a backoff.
	175	+ //
	176	+ // See the guidelines above for deciding between `FAILED_PRECONDITION`,
	177	+ // `ABORTED`, and `UNAVAILABLE`.
	178	+ //
	179	+ // HTTP Mapping: 503 Service Unavailable
	180	+ UNAVAILABLE = 14;
	181	+
	182	+ // Unrecoverable data loss or corruption.
	183	+ //
	184	+ // HTTP Mapping: 500 Internal Server Error
	185	+ DATA_LOSS = 15;
	186	+}

google/rpc/code_pb2.py

---

	1	+# Generated by the protocol buffer compiler. DO NOT EDIT!
	2	+# source: google/rpc/code.proto
	3	+
	4	+import sys
	5	+_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
	6	+from google.protobuf.internal import enum_type_wrapper
	7	+from google.protobuf import descriptor as _descriptor
	8	+from google.protobuf import message as _message
	9	+from google.protobuf import reflection as _reflection
	10	+from google.protobuf import symbol_database as _symbol_database
	11	+from google.protobuf import descriptor_pb2
	12	+# @@protoc_insertion_point(imports)
	13	+
	14	+_sym_db = _symbol_database.Default()
	15	+
	16	+
	17	+
	18	+
	19	+DESCRIPTOR = _descriptor.FileDescriptor(
	20	+ name='google/rpc/code.proto',
	21	+ package='google.rpc',
	22	+ syntax='proto3',
	23	+ serialized_pb=_b('\n\x15google/rpc/code.proto\x12\ngoogle.rpc*\xb7\x02\n\x04\x43ode\x12\x06\n\x02OK\x10\x00\x12\r\n\tCANCELLED\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x14\n\x10INVALID_ARGUMENT\x10\x03\x12\x15\n\x11\x44\x45\x41\x44LINE_EXCEEDED\x10\x04\x12\r\n\tNOT_FOUND\x10\x05\x12\x12\n\x0e\x41LREADY_EXISTS\x10\x06\x12\x15\n\x11PERMISSION_DENIED\x10\x07\x12\x13\n\x0fUNAUTHENTICATED\x10\x10\x12\x16\n\x12RESOURCE_EXHAUSTED\x10\x08\x12\x17\n\x13\x46\x41ILED_PRECONDITION\x10\t\x12\x0b\n\x07\x41\x42ORTED\x10\n\x12\x10\n\x0cOUT_OF_RANGE\x10\x0b\x12\x11\n\rUNIMPLEMENTED\x10\x0c\x12\x0c\n\x08INTERNAL\x10\r\x12\x0f\n\x0bUNAVAILABLE\x10\x0e\x12\r\n\tDATA_LOSS\x10\x0f\x42X\n\x0e\x63om.google.rpcB\tCodeProtoP\x01Z3google.golang.org/genproto/googleapis/rpc/code;code\xa2\x02\x03RPCb\x06proto3')
	24	+)
	25	+
	26	+_CODE = _descriptor.EnumDescriptor(
	27	+ name='Code',
	28	+ full_name='google.rpc.Code',
	29	+ filename=None,
	30	+ file=DESCRIPTOR,
	31	+ values=[
	32	+ _descriptor.EnumValueDescriptor(
	33	+ name='OK', index=0, number=0,
	34	+ options=None,
	35	+ type=None),
	36	+ _descriptor.EnumValueDescriptor(
	37	+ name='CANCELLED', index=1, number=1,
	38	+ options=None,
	39	+ type=None),
	40	+ _descriptor.EnumValueDescriptor(
	41	+ name='UNKNOWN', index=2, number=2,
	42	+ options=None,
	43	+ type=None),
	44	+ _descriptor.EnumValueDescriptor(
	45	+ name='INVALID_ARGUMENT', index=3, number=3,
	46	+ options=None,
	47	+ type=None),
	48	+ _descriptor.EnumValueDescriptor(
	49	+ name='DEADLINE_EXCEEDED', index=4, number=4,
	50	+ options=None,
	51	+ type=None),
	52	+ _descriptor.EnumValueDescriptor(
	53	+ name='NOT_FOUND', index=5, number=5,
	54	+ options=None,
	55	+ type=None),
	56	+ _descriptor.EnumValueDescriptor(
	57	+ name='ALREADY_EXISTS', index=6, number=6,
	58	+ options=None,
	59	+ type=None),
	60	+ _descriptor.EnumValueDescriptor(
	61	+ name='PERMISSION_DENIED', index=7, number=7,
	62	+ options=None,
	63	+ type=None),
	64	+ _descriptor.EnumValueDescriptor(
	65	+ name='UNAUTHENTICATED', index=8, number=16,
	66	+ options=None,
	67	+ type=None),
	68	+ _descriptor.EnumValueDescriptor(
	69	+ name='RESOURCE_EXHAUSTED', index=9, number=8,
	70	+ options=None,
	71	+ type=None),
	72	+ _descriptor.EnumValueDescriptor(
	73	+ name='FAILED_PRECONDITION', index=10, number=9,
	74	+ options=None,
	75	+ type=None),
	76	+ _descriptor.EnumValueDescriptor(
	77	+ name='ABORTED', index=11, number=10,
	78	+ options=None,
	79	+ type=None),
	80	+ _descriptor.EnumValueDescriptor(
	81	+ name='OUT_OF_RANGE', index=12, number=11,
	82	+ options=None,
	83	+ type=None),
	84	+ _descriptor.EnumValueDescriptor(
	85	+ name='UNIMPLEMENTED', index=13, number=12,
	86	+ options=None,
	87	+ type=None),
	88	+ _descriptor.EnumValueDescriptor(
	89	+ name='INTERNAL', index=14, number=13,
	90	+ options=None,
	91	+ type=None),
	92	+ _descriptor.EnumValueDescriptor(
	93	+ name='UNAVAILABLE', index=15, number=14,
	94	+ options=None,
	95	+ type=None),
	96	+ _descriptor.EnumValueDescriptor(
	97	+ name='DATA_LOSS', index=16, number=15,
	98	+ options=None,
	99	+ type=None),
	100	+ ],
	101	+ containing_type=None,
	102	+ options=None,
	103	+ serialized_start=38,
	104	+ serialized_end=349,
	105	+)
	106	+_sym_db.RegisterEnumDescriptor(_CODE)
	107	+
	108	+Code = enum_type_wrapper.EnumTypeWrapper(_CODE)
	109	+OK = 0
	110	+CANCELLED = 1
	111	+UNKNOWN = 2
	112	+INVALID_ARGUMENT = 3
	113	+DEADLINE_EXCEEDED = 4
	114	+NOT_FOUND = 5
	115	+ALREADY_EXISTS = 6
	116	+PERMISSION_DENIED = 7
	117	+UNAUTHENTICATED = 16
	118	+RESOURCE_EXHAUSTED = 8
	119	+FAILED_PRECONDITION = 9
	120	+ABORTED = 10
	121	+OUT_OF_RANGE = 11
	122	+UNIMPLEMENTED = 12
	123	+INTERNAL = 13
	124	+UNAVAILABLE = 14
	125	+DATA_LOSS = 15
	126	+
	127	+
	128	+DESCRIPTOR.enum_types_by_name['Code'] = _CODE
	129	+_sym_db.RegisterFileDescriptor(DESCRIPTOR)
	130	+
	131	+
	132	+DESCRIPTOR.has_options = True
	133	+DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\n\016com.google.rpcB\tCodeProtoP\001Z3google.golang.org/genproto/googleapis/rpc/code;code\242\002\003RPC'))
	134	+# @@protoc_insertion_point(module_scope)

google/rpc/code_pb2_grpc.py

---

	1	+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
	2	+import grpc
	3	+

tests/integration/__init__.py

---

tests/integration/bots_service.py

---

...	...	@@ -131,24 +131,6 @@ def test_update_leases(number_of_leases, bot_session, context, instance):
131	131	assert len(response.leases) == len(bot.leases)
132	132	assert bot == response
133	133
134		-@pytest.mark.parametrize("number_of_leases", [3])
135		-def test_update_leases_out_of_sync(number_of_leases, bot_session, context, instance):
136		- leases = [bots_pb2.Lease() for x in range(number_of_leases)]
137		- bot_session.leases.extend(leases)
138		- request = bots_pb2.CreateBotSessionRequest(parent='',
139		- bot_session=bot_session)
140		- # Simulated the severed binding between client and server
141		- bot = copy.deepcopy(instance.CreateBotSession(request, context))
142		- bot.leases.extend(leases)
143		-
144		- request = bots_pb2.UpdateBotSessionRequest(name=bot.name,
145		- bot_session=bot)
146		-
147		- response = instance.UpdateBotSession(request, context)
148		-
149		- assert isinstance(response, bots_pb2.BotSession)
150		- context.set_code.assert_called_once_with(grpc.StatusCode.DATA_LOSS)
151		-
152	134	def test_update_leases_with_work(bot_session, context, instance):
153	135	leases = [bots_pb2.Lease() for x in range(2)]
154	136	bot_session.leases.extend(leases)

tests/job.py

---

...	...	@@ -65,25 +65,21 @@ def test_get_operation_meta(mock_meta, instance):
65	65
66	66	@mock.patch.object(job.bots_pb2, 'Lease', autospec = True)
67	67	@mock.patch.object(job.Job,'_pack_any', autospec = True)
68		-def test_get_lease(mock_pack, mock_lease, instance):
	68	+def test_create_lease(mock_pack, mock_lease, instance):
69	69	action='harry'
70	70	name = 'bryant'
71		- lease_state = LeaseState.COMPLETED
	71	+ lease_state = LeaseState.PENDING
72	72
73	73	instance.action = action
74	74	instance.name = name
75	75	instance.lease_state = lease_state
76	76
77		- assert instance.get_lease() is mock_lease.return_value
	77	+ assert instance.create_lease() is mock_lease.return_value
78	78	mock_pack.assert_called_once_with(instance, action)
79	79	mock_lease.assert_called_once_with(assignment=name,
80	80	inline_assignment=mock_pack.return_value,
81	81	state=lease_state.value)
82	82
83		-def test_cancel():
84		- # TODO: Implement cancel
85		- pass
86		-
87	83	@mock.patch.object(job.Job, 'get_operation', autospec = True)
88	84	@mock.patch.object(job.operations_pb2,'ListOperationsResponse', autospec = True)
89	85	def test_get_operations(mock_response, mock_get_operation, instance):

tests/scheduler.py

---

...	...	@@ -66,20 +66,19 @@ def test_retry_job_fail(instance):
66	66
67	67	assert mock_job.execute_stage is ExecuteStage.COMPLETED
68	68	assert mock_job.n_tries is n_tries
69		- mock_job.cancel.assert_called_once()
70	69	instance.jobs.__setitem__.assert_called_once_with(name, mock_job)
71	70
72		-def test_get_job(instance):
	71	+def test_create_job(instance):
73	72	name = 'eldon'
74	73
75	74	mock_job = mock.MagicMock()
76	75	mock_job.name = name
77	76	instance.queue.popleft.return_value = mock_job
	77	+ instance.queue.__len__.return_value = 1
78	78
79		- assert instance.get_job() is mock_job
	79	+ assert instance.create_job() is mock_job
80	80
81	81	assert mock_job.execute_stage is ExecuteStage.EXECUTING
82		- assert mock_job.lease_state is LeaseState.PENDING
83	82	instance.jobs.__setitem__.assert_called_once_with(name, mock_job)
84	83
85	84	def test_job_complete(instance):
...	...	@@ -97,6 +96,72 @@ def test_job_complete(instance):
97	96	instance.jobs.__getitem__.assert_called_once_with(name)
98	97	instance.jobs.__setitem__.assert_called_once_with(name, mock_job)
99	98
	99	+@pytest.mark.parametrize("state", [LeaseState.LEASE_STATE_UNSPECIFIED,
	100	+ LeaseState.PENDING,
	101	+ LeaseState.ACTIVE,
	102	+ LeaseState.COMPLETED,
	103	+ mock.Mock(value = 100)
	104	+ ])
	105	+@mock.patch.object(scheduler.Scheduler,'job_complete', autospec = True)
	106	+@mock.patch.object(scheduler.Scheduler,'create_job', autospec = True)
	107	+def test_update_lease_state_with_work(mock_create_job, mock_job_complete, state, instance):
	108	+ name = 'orion'
	109	+
	110	+ mock_lease = mock.Mock()
	111	+ mock_lease.assignment = name
	112	+ mock_lease.state = state.value
	113	+
	114	+ if state == LeaseState.LEASE_STATE_UNSPECIFIED or \
	115	+ state == LeaseState.COMPLETED:
	116	+ assert instance.update_lease(mock_lease) is mock_create_job.return_value.lease
	117	+ mock_create_job.assert_called_once_with(instance)
	118	+ mock_create_job.return_value.create_lease.assert_called_once_with()
	119	+ instance.jobs.__setitem__.assert_called_once_with(name, mock_create_job.return_value)
	120	+
	121	+ elif state == LeaseState.PENDING or \
	122	+ state == LeaseState.ACTIVE or \
	123	+ state == LeaseState.CANCELLED:
	124	+ assert instance.update_lease(mock_lease) is mock_lease
	125	+
	126	+ else:
	127	+ with pytest.raises(Exception):
	128	+ instance.update_lease(mock_lease)
	129	+
	130	+ instance.jobs.get.assert_called_once_with(name)
	131	+
	132	+@pytest.mark.parametrize("state", [LeaseState.LEASE_STATE_UNSPECIFIED,
	133	+ LeaseState.PENDING,
	134	+ LeaseState.ACTIVE,
	135	+ LeaseState.COMPLETED,
	136	+ mock.Mock(value = 100)
	137	+ ])
	138	+@mock.patch.object(scheduler.Scheduler,'job_complete', autospec = True)
	139	+@mock.patch.object(scheduler.Scheduler,'create_job', autospec = True)
	140	+def test_update_lease_state_without_work(mock_create_job, mock_job_complete, state, instance):
	141	+ name = 'orion'
	142	+
	143	+ mock_lease = mock.Mock()
	144	+ mock_lease.assignment = name
	145	+ mock_lease.state = state.value
	146	+
	147	+ mock_create_job.return_value = None
	148	+
	149	+ if state == LeaseState.LEASE_STATE_UNSPECIFIED or \
	150	+ state == LeaseState.COMPLETED:
	151	+ assert instance.update_lease(mock_lease) is mock_lease
	152	+ mock_create_job.assert_called_once_with(instance)
	153	+
	154	+ elif state == LeaseState.PENDING or \
	155	+ state == LeaseState.ACTIVE or \
	156	+ state == LeaseState.CANCELLED:
	157	+ assert instance.update_lease(mock_lease) is mock_lease
	158	+
	159	+ else:
	160	+ with pytest.raises(Exception):
	161	+ instance.update_lease(mock_lease)
	162	+
	163	+ instance.jobs.get.assert_called_once_with(name)
	164	+
100	165	@mock.patch.object(scheduler, 'operations_pb2', autospec = True)
101	166	def test_get_operations(mock_pb2, instance):
102	167	value = 'eldon'
...	...	@@ -112,4 +177,4 @@ def test_get_operations(mock_pb2, instance):
112	177	assert instance.get_operations() is response.return_value
113	178	response_value.get_operation.assert_called_once()
114	179	response.return_value.operations.extend.assert_called_once_with([value])
115		- response.assert_called_once()
	180	+ response.assert_called_once_with()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.