[Notes] [Git][BuildGrid/buildgrid][santi/155-repeated-digest-status-msgs-fix] 5 commits: server/instance.py: Deactivate SO_REUSEPORT option

[Santiago Gil <gitlab mg gitlab com>]

Title: GitLab

Santiago Gil pushed to branch santi/155-repeated-digest-status-msgs-fix at BuildGrid / buildgrid

Commits:

• 15d44d3b
  by Martin Blanchard at 2019-01-24T08:57:04Z

  server/instance.py: Deactivate SO_REUSEPORT option
  
  https://gitlab.com/BuildGrid/buildgrid/issues/138
  

• 4a600cfc
  by Martin Blanchard at 2019-01-24T08:57:04Z

  _exceptions.py: Add a PermissionDeniedError
  
  https://gitlab.com/BuildGrid/buildgrid/issues/138
  

• 89f2b765
  by Martin Blanchard at 2019-01-24T08:57:04Z

  server/instance.py: Raise an error on socket binding failure
  
  https://gitlab.com/BuildGrid/buildgrid/issues/138
  

• 02692ded
  by Martin Blanchard at 2019-01-24T08:57:04Z

  cmd_server.py: Catch socket configuration errors
  
  https://gitlab.com/BuildGrid/buildgrid/issues/138
  

• 852fc015
  by Santiago Gil at 2019-01-24T13:42:36Z

  upload_{file, directory}(): fix status messages
  
  When uploading items with the same digest but different paths, the
  status messages referred only to one of those filenames.
  
  These changes fix that, considering each of those different files in the
  status messages printed to stdout.
  
  Also, upload_file() and upload_directory() now print paths consistently.
  
  https://gitlab.com/BuildGrid/buildgrid/issues/155
  

4 changed files:

• buildgrid/_app/commands/cmd_cas.py
• buildgrid/_app/commands/cmd_server.py
• buildgrid/_exceptions.py
• buildgrid/server/instance.py

Changes:

buildgrid/_app/commands/cmd_cas.py

---

...	...	@@ -82,20 +82,19 @@ def upload_dummy(context):
82	82	help="Check uploaded files integrity.")
83	83	@pass_context
84	84	def upload_file(context, file_path, verify):
85		- sent_digests, files_map = [], {}
	85	+ sent_digests = []
86	86	with upload(context.channel, instance=context.instance_name) as uploader:
87	87	for path in file_path:
88	88	if not os.path.isabs(path):
89		- path = os.path.abspath(path)
	89	+ path = os.path.relpath(path)
	90	+
90	91	click.echo("Queueing path=[{}]".format(path))
91	92
92	93	file_digest = uploader.upload_file(path, queue=True)
93	94
94		- files_map[file_digest.hash] = path
95		- sent_digests.append(file_digest)
	95	+ sent_digests.append((file_digest, path))
96	96
97		- for file_digest in sent_digests:
98		- file_path = os.path.relpath(files_map[file_digest.hash])
	97	+ for file_digest, file_path in sent_digests:
99	98	if verify and file_digest.size_bytes != os.stat(file_path).st_size:
100	99	click.echo("Error: Failed to verify '{}'".format(file_path), err=True)
101	100	elif file_digest.ByteSize():
...	...	@@ -111,22 +110,17 @@ def upload_file(context, file_path, verify):
111	110	help="Check uploaded directory's integrity.")
112	111	@pass_context
113	112	def upload_directory(context, directory_path, verify):
114		- sent_digests, nodes_map = [], {}
	113	+ sent_digests = []
115	114	with upload(context.channel, instance=context.instance_name) as uploader:
116	115	for node, blob, path in merkle_tree_maker(directory_path):
117		- if not os.path.isabs(path):
118		- path = os.path.abspath(path)
	116	+ if not os.path.isabs(directory_path):
	117	+ path = os.path.relpath(path)
119	118	click.echo("Queueing path=[{}]".format(path))
120	119
121	120	node_digest = uploader.put_blob(blob, digest=node.digest, queue=True)
	121	+ sent_digests.append((node_digest, path))
122	122
123		- nodes_map[node.digest.hash] = path
124		- sent_digests.append(node_digest)
125		-
126		- for node_digest in sent_digests:
127		- node_path = nodes_map[node_digest.hash]
128		- if not os.path.isabs(directory_path):
129		- node_path = os.path.relpath(node_path)
	123	+ for node_digest, node_path in sent_digests:
130	124	if verify and (os.path.isfile(node_path) and
131	125	node_digest.size_bytes != os.stat(node_path).st_size):
132	126	click.echo("Error: Failed to verify path=[{}]".format(node_path), err=True)

buildgrid/_app/commands/cmd_server.py

---

...	...	@@ -24,6 +24,7 @@ import sys
24	24
25	25	import click
26	26
	27	+from buildgrid._exceptions import PermissionDeniedError
27	28	from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm
28	29	from buildgrid.server.instance import BuildGridServer
29	30	from buildgrid.server._monitoring import MonitoringOutputType, MonitoringOutputFormat
...	...	@@ -120,8 +121,13 @@ def _create_server_from_config(configuration):
120	121
121	122	server = BuildGridServer(**kargs)
122	123
123		- for channel in network:
124		- server.add_port(channel.address, channel.credentials)
	124	+ try:
	125	+ for channel in network:
	126	+ server.add_port(channel.address, channel.credentials)
	127	+
	128	+ except PermissionDeniedError as e:
	129	+ click.echo("Error: {}.".format(e), err=True)
	130	+ sys.exit(-1)
125	131
126	132	for instance in instances:
127	133	instance_name = instance['name']

buildgrid/_exceptions.py

---

...	...	@@ -89,3 +89,9 @@ class FailedPreconditionError(BgdError):
89	89	able to fix the errors and retry."""
90	90	def __init__(self, message, detail=None, reason=None):
91	91	super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)
	92	+
	93	+
	94	+class PermissionDeniedError(BgdError):
	95	+ """The caller does not have permission to execute the specified operation."""
	96	+ def __init__(self, message, detail=None, reason=None):
	97	+ super().__init__(message, detail=detail, domain=ErrorDomain.SERVER, reason=reason)

buildgrid/server/instance.py

---

...	...	@@ -27,6 +27,7 @@ import grpc
27	27	import janus
28	28
29	29	from buildgrid._enums import BotStatus, LogRecordLevel, MetricRecordDomain, MetricRecordType
	30	+from buildgrid._exceptions import PermissionDeniedError
30	31	from buildgrid._protos.buildgrid.v2 import monitoring_pb2
31	32	from buildgrid.server.actioncache.service import ActionCacheService
32	33	from buildgrid.server._authentication import AuthMetadataMethod, AuthMetadataAlgorithm
...	...	@@ -87,7 +88,8 @@ class BuildGridServer:
87	88	AuthContext.interceptor = self.__grpc_auth_interceptor
88	89
89	90	self.__grpc_executor = futures.ThreadPoolExecutor(max_workers)
90		- self.__grpc_server = grpc.server(self.__grpc_executor)
	91	+ self.__grpc_server = grpc.server(self.__grpc_executor,
	92	+ options=(('grpc.so_reuseport', 0),))
91	93
92	94	self.__main_loop = asyncio.get_event_loop()
93	95
...	...	@@ -205,6 +207,9 @@ class BuildGridServer:
205	207
206	208	Returns:
207	209	int: Number of the bound port.
	210	+
	211	+ Raises:
	212	+ PermissionDeniedError: If socket binding fails.
208	213	"""
209	214	if credentials is not None:
210	215	self.__logger.info("Adding secure connection on: [%s]", address)
...	...	@@ -214,6 +219,9 @@ class BuildGridServer:
214	219	self.__logger.info("Adding insecure connection on [%s]", address)
215	220	port_number = self.__grpc_server.add_insecure_port(address)
216	221
	222	+ if not port_number:
	223	+ raise PermissionDeniedError("Unable to configure socket")
	224	+
217	225	return port_number
218	226
219	227	def add_execution_instance(self, instance, instance_name):

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.