[Notes] [Git][BuildStream/buildstream][valentindavid/ruamel_0_15_41] 7 commits: tests: Avoid hangs due to exceptions in subprocesses

[Valentin David <gitlab mg gitlab com>]

Title: GitLab

Valentin David pushed to branch valentindavid/ruamel_0_15_41 at BuildStream / buildstream

Commits:

• 4ca892e3
  by Jim MacArthur at 2018-11-27T09:30:13Z

  tests: Avoid hangs due to exceptions in subprocesses
  

• d09a18b8
  by Jim MacArthur at 2018-11-27T10:31:07Z

  Merge branch 'jmac/fix-test-hangs-2' into 'master'
  
  Avoid hanging artifact cache tests
  
  See merge request BuildStream/buildstream!964

• df60258a
  by Jürg Billeter at 2018-11-27T12:11:24Z

  source.py: Add optional _get_local_path() method
  

• 42809933
  by Jürg Billeter at 2018-11-27T12:11:24Z

  local.py: Implement _get_local_path()
  

• eb5b10f7
  by Jürg Billeter at 2018-11-27T12:11:24Z

  _loader.py: Use Source._get_local_path() for subprojects
  
  This eliminates unneeded copies of the project directory for junctions
  with a single local source.
  

• b2ea208c
  by Jürg Billeter at 2018-11-27T12:50:46Z

  Merge branch 'juerg/local-junctions' into 'master'
  
  Optimization for local junctions
  
  See merge request BuildStream/buildstream!290

• c3b00baf
  by Valentin David at 2018-11-27T13:53:44Z

  setup.py: require ruamel.yaml >= 0.15.41 < 0.15.52
  

7 changed files:

• buildstream/_loader/loader.py
• buildstream/plugins/sources/local.py
• buildstream/source.py
• setup.py
• tests/artifactcache/pull.py
• tests/artifactcache/push.py
• tests/testutils/artifactshare.py

Changes:

buildstream/_loader/loader.py

---

...	...	@@ -563,17 +563,23 @@ class Loader():
563	563	"Subproject has no ref for junction: {}".format(filename),
564	564	detail=detail)
565	565
566		- # Stage sources
567		- os.makedirs(self._context.builddir, exist_ok=True)
568		- basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self._context.builddir)
569		- element._stage_sources_at(basedir, mount_workspaces=False)
	566	+ if len(sources) == 1 and sources[0]._get_local_path():
	567	+ # Optimization for junctions with a single local source
	568	+ basedir = sources[0]._get_local_path()
	569	+ tempdir = None
	570	+ else:
	571	+ # Stage sources
	572	+ os.makedirs(self._context.builddir, exist_ok=True)
	573	+ basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self._context.builddir)
	574	+ element._stage_sources_at(basedir, mount_workspaces=False)
	575	+ tempdir = basedir
570	576
571	577	# Load the project
572	578	project_dir = os.path.join(basedir, element.path)
573	579	try:
574	580	from .._project import Project
575	581	project = Project(project_dir, self._context, junction=element,
576		- parent_loader=self, tempdir=basedir)
	582	+ parent_loader=self, tempdir=tempdir)
577	583	except LoadError as e:
578	584	if e.reason == LoadErrorReason.MISSING_PROJECT_CONF:
579	585	raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION,

buildstream/plugins/sources/local.py

---

...	...	@@ -124,6 +124,9 @@ class LocalSource(Source):
124	124	else:
125	125	os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH)
126	126
	127	+ def _get_local_path(self):
	128	+ return self.fullpath
	129	+
127	130
128	131	# Create a unique key for a file
129	132	def unique_key(filename):

buildstream/source.py

---

...	...	@@ -615,6 +615,23 @@ class Source(Plugin):
615	615	with utils._tempdir(dir=mirrordir) as tempdir:
616	616	yield tempdir
617	617
	618	+ #############################################################
	619	+ # Private Abstract Methods used in BuildStream #
	620	+ #############################################################
	621	+
	622	+ # Returns the local path to the source
	623	+ #
	624	+ # If the source is locally available, this method returns the absolute
	625	+ # path. Otherwise, the return value is None.
	626	+ #
	627	+ # This is an optimization for local sources and optional to implement.
	628	+ #
	629	+ # Returns:
	630	+ # (str): The local absolute path, or None
	631	+ #
	632	+ def _get_local_path(self):
	633	+ return None
	634	+
618	635	#############################################################
619	636	# Private Methods used in BuildStream #
620	637	#############################################################

setup.py

---

...	...	@@ -337,7 +337,14 @@ setup(name='BuildStream',
337	337	install_requires=[
338	338	'setuptools',
339	339	'psutil',
340		- 'ruamel.yaml < 0.15.52',
	340	+ # According to ruamel.yaml's PyPI page, we are suppose to use
	341	+ # "<=0.15" in production until 0.15 becomes API stable.
	342	+ # However we need ruamel.yaml 0.15.41 or greater for Python 3.7.
	343	+ # We know that ruamel.yaml 0.15.52 breaks API in a way that
	344	+ # is incompatible with BuildStream.
	345	+ #
	346	+ # See issues #571 and #790.
	347	+ 'ruamel.yaml >= 0.15.41, < 0.15.52',
341	348	'pluginbase',
342	349	'Click',
343	350	'jinja2 >= 2.10',

tests/artifactcache/pull.py

---

...	...	@@ -25,6 +25,17 @@ def message_handler(message, context):
25	25	pass
26	26
27	27
	28	+# Since parent processes wait for queue events, we need
	29	+# to put something on it if the called process raises an
	30	+# exception.
	31	+def _queue_wrapper(target, queue, *args):
	32	+ try:
	33	+ target(*args, queue=queue)
	34	+ except Exception as e:
	35	+ queue.put(str(e))
	36	+ raise
	37	+
	38	+
28	39	def tree_maker(cas, tree, directory):
29	40	if tree.root.ByteSize() == 0:
30	41	tree.root.CopyFrom(directory)
...	...	@@ -97,9 +108,9 @@ def test_pull(cli, tmpdir, datafiles):
97	108	queue = multiprocessing.Queue()
98	109	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
99	110	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
100		- process = multiprocessing.Process(target=_test_pull,
101		- args=(user_config_file, project_dir, artifact_dir,
102		- 'target.bst', element_key, queue))
	111	+ process = multiprocessing.Process(target=_queue_wrapper,
	112	+ args=(_test_pull, queue, user_config_file, project_dir,
	113	+ artifact_dir, 'target.bst', element_key))
103	114
104	115	try:
105	116	# Keep SIGINT blocked in the child process
...	...	@@ -205,9 +216,9 @@ def test_pull_tree(cli, tmpdir, datafiles):
205	216	queue = multiprocessing.Queue()
206	217	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
207	218	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
208		- process = multiprocessing.Process(target=_test_push_tree,
209		- args=(user_config_file, project_dir, artifact_dir,
210		- artifact_digest, queue))
	219	+ process = multiprocessing.Process(target=_queue_wrapper,
	220	+ args=(_test_push_tree, queue, user_config_file, project_dir,
	221	+ artifact_dir, artifact_digest))
211	222
212	223	try:
213	224	# Keep SIGINT blocked in the child process
...	...	@@ -233,9 +244,9 @@ def test_pull_tree(cli, tmpdir, datafiles):
233	244
234	245	queue = multiprocessing.Queue()
235	246	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
236		- process = multiprocessing.Process(target=_test_pull_tree,
237		- args=(user_config_file, project_dir, artifact_dir,
238		- tree_digest, queue))
	247	+ process = multiprocessing.Process(target=_queue_wrapper,
	248	+ args=(_test_pull_tree, queue, user_config_file, project_dir,
	249	+ artifact_dir, tree_digest))
239	250
240	251	try:
241	252	# Keep SIGINT blocked in the child process

tests/artifactcache/push.py

---

...	...	@@ -26,6 +26,17 @@ def message_handler(message, context):
26	26	pass
27	27
28	28
	29	+# Since parent processes wait for queue events, we need
	30	+# to put something on it if the called process raises an
	31	+# exception.
	32	+def _queue_wrapper(target, queue, *args):
	33	+ try:
	34	+ target(*args, queue=queue)
	35	+ except Exception as e:
	36	+ queue.put(str(e))
	37	+ raise
	38	+
	39	+
29	40	@pytest.mark.datafiles(DATA_DIR)
30	41	def test_push(cli, tmpdir, datafiles):
31	42	project_dir = str(datafiles)
...	...	@@ -76,9 +87,9 @@ def test_push(cli, tmpdir, datafiles):
76	87	queue = multiprocessing.Queue()
77	88	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
78	89	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
79		- process = multiprocessing.Process(target=_test_push,
80		- args=(user_config_file, project_dir, artifact_dir,
81		- 'target.bst', element_key, queue))
	90	+ process = multiprocessing.Process(target=_queue_wrapper,
	91	+ args=(_test_push, queue, user_config_file, project_dir,
	92	+ artifact_dir, 'target.bst', element_key))
82	93
83	94	try:
84	95	# Keep SIGINT blocked in the child process
...	...	@@ -185,9 +196,9 @@ def test_push_directory(cli, tmpdir, datafiles):
185	196	queue = multiprocessing.Queue()
186	197	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
187	198	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
188		- process = multiprocessing.Process(target=_test_push_directory,
189		- args=(user_config_file, project_dir, artifact_dir,
190		- artifact_digest, queue))
	199	+ process = multiprocessing.Process(target=_queue_wrapper,
	200	+ args=(_test_push_directory, queue, user_config_file,
	201	+ project_dir, artifact_dir, artifact_digest))
191	202
192	203	try:
193	204	# Keep SIGINT blocked in the child process
...	...	@@ -260,8 +271,9 @@ def test_push_message(cli, tmpdir, datafiles):
260	271	queue = multiprocessing.Queue()
261	272	# Use subprocess to avoid creation of gRPC threads in main BuildStream process
262	273	# See https://github.com/grpc/grpc/blob/master/doc/fork_support.md for details
263		- process = multiprocessing.Process(target=_test_push_message,
264		- args=(user_config_file, project_dir, artifact_dir, queue))
	274	+ process = multiprocessing.Process(target=_queue_wrapper,
	275	+ args=(_test_push_message, queue, user_config_file,
	276	+ project_dir, artifact_dir))
265	277
266	278	try:
267	279	# Keep SIGINT blocked in the child process

tests/testutils/artifactshare.py

---

...	...	@@ -67,19 +67,24 @@ class ArtifactShare():
67	67	def run(self, q):
68	68	pytest_cov.embed.cleanup_on_sigterm()
69	69
70		- # Optionally mock statvfs
71		- if self.total_space:
72		- if self.free_space is None:
73		- self.free_space = self.total_space
74		- os.statvfs = self._mock_statvfs
	70	+ try:
	71	+ # Optionally mock statvfs
	72	+ if self.total_space:
	73	+ if self.free_space is None:
	74	+ self.free_space = self.total_space
	75	+ os.statvfs = self._mock_statvfs
	76	+
	77	+ server = create_server(self.repodir, enable_push=True)
	78	+ port = server.add_insecure_port('localhost:0')
75	79
76		- server = create_server(self.repodir, enable_push=True)
77		- port = server.add_insecure_port('localhost:0')
	80	+ server.start()
78	81
79		- server.start()
	82	+ # Send port to parent
	83	+ q.put(port)
80	84
81		- # Send port to parent
82		- q.put(port)
	85	+ except Exception as e:
	86	+ q.put(None)
	87	+ raise
83	88
84	89	# Sleep until termination by signal
85	90	signal.pause()

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.