[Notes] [Git][BuildStream/buildstream][richardmaw/test-config-fixes] 5 commits: Handle sockets when copying trees into artifacts

[richardmaw-codethink <gitlab mg gitlab com>]

Title: GitLab

richardmaw-codethink pushed to branch richardmaw/test-config-fixes at BuildStream / buildstream

Commits:

• 233a7d83
  by Richard Maw at 2018-09-14T08:53:14Z

  Handle sockets when copying trees into artifacts
  
  We can't include a socket in a CAS tree, but it's mostly meaningless to do so
  since there can't possibly be a process serving it.
  

• f06f234a
  by Richard Maw at 2018-09-14T08:53:14Z

  tests: cover builds that make sockets
  

• f86ab8f6
  by richardmaw-codethink at 2018-09-14T09:46:07Z

  Merge branch 'richardmaw/builddir-sockets' into 'master'
  
  Fix: While caching build artifact: "Cannot extract [path to socket file] into staging-area. Unsupported type."
  
  See merge request BuildStream/buildstream!783

• e7427462
  by Richard Maw at 2018-09-14T10:28:17Z

  tests: Use cli.configure for custom logging test
  
  Overriding the config with a custom config file on the command-line
  prevents it merging in the test-specific configuration
  and can permit it to attempt to initialise the user's cache.
  

• 800a8403
  by Richard Maw at 2018-09-14T10:28:17Z

  Tidy relative workspace tests
  
  They were moving the whole tmpdir to move the project repository.
  This moves the cache directories etc. as well, meaning cli.run can't find them.
  This was worked around by setting configure=False,
  but this has the side-effect of making use of the user's caches,
  which it should not be doing for reproducibility reasons.
  
  By changing the tempdir layout to have the project in a subdirectory
  we can move the project around for the relative workspace tests
  without losing track of the configured state directories,
  so we can leave configure=True and avoid touching the user's caches.
  

7 changed files:

• buildstream/_artifactcache/cascache.py
• buildstream/utils.py
• tests/frontend/logging.py
• tests/frontend/workspace.py
• + tests/integration/project/elements/sockets/make-builddir-socket.bst
• + tests/integration/project/elements/sockets/make-install-root-socket.bst
• + tests/integration/sockets.py

Changes:

buildstream/_artifactcache/cascache.py

---

...	...	@@ -684,6 +684,9 @@ class CASCache(ArtifactCache):
684	684	symlinknode = directory.symlinks.add()
685	685	symlinknode.name = name
686	686	symlinknode.target = os.readlink(full_path)
	687	+ elif stat.S_ISSOCK(mode):
	688	+ # The process serving the socket can't be cached anyway
	689	+ pass
687	690	else:
688	691	raise ArtifactError("Unsupported file type for {}".format(full_path))
689	692

buildstream/utils.py

---

...	...	@@ -372,6 +372,8 @@ def copy_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
372	372	Directories in `dest` are replaced with files from `src`,
373	373	unless the existing directory in `dest` is not empty in which
374	374	case the path will be reported in the return value.
	375	+
	376	+ UNIX domain socket files from `src` are ignored.
375	377	"""
376	378	presorted = False
377	379	if files is None:
...	...	@@ -414,6 +416,8 @@ def link_files(src, dest, *, files=None, ignore_missing=False, report_written=Fa
414	416
415	417	If a hardlink cannot be created due to crossing filesystems,
416	418	then the file will be copied instead.
	419	+
	420	+ UNIX domain socket files from `src` are ignored.
417	421	"""
418	422	presorted = False
419	423	if files is None:
...	...	@@ -841,6 +845,13 @@ def _process_list(srcdir, destdir, filelist, actionfunc, result,
841	845	os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
842	846	os.chmod(destpath, file_stat.st_mode)
843	847
	848	+ elif stat.S_ISFIFO(mode):
	849	+ os.mkfifo(destpath, mode)
	850	+
	851	+ elif stat.S_ISSOCK(mode):
	852	+ # We can't duplicate the process serving the socket anyway
	853	+ pass
	854	+
844	855	else:
845	856	# Unsupported type.
846	857	raise UtilError('Cannot extract {} into staging-area. Unsupported type.'.format(srcpath))

tests/frontend/logging.py

---

...	...	@@ -54,8 +54,7 @@ def test_custom_logging(cli, tmpdir, datafiles):
54	54
55	55	custom_log_format = '%{elapsed},%{elapsed-us},%{wallclock},%{key},%{element},%{action},%{message}'
56	56	user_config = {'logging': {'message-format': custom_log_format}}
57		- user_config_file = str(tmpdir.join('buildstream.conf'))
58		- _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file)
	57	+ cli.configure(user_config)
59	58
60	59	# Create our repo object of the given source type with
61	60	# the bin files, and then collect the initial ref.
...	...	@@ -75,7 +74,7 @@ def test_custom_logging(cli, tmpdir, datafiles):
75	74	element_name))
76	75
77	76	# Now try to fetch it
78		- result = cli.run(project=project, args=['-c', user_config_file, 'fetch', element_name])
	77	+ result = cli.run(project=project, args=['fetch', element_name])
79	78	result.assert_success()
80	79
81	80	m = re.search("\d\d:\d\d:\d\d,\d\d:\d\d:\d\d.\d{6},\d\d:\d\d:\d\d,,,SUCCESS,Checking sources", result.stderr)

tests/frontend/workspace.py

---

...	...	@@ -43,10 +43,13 @@ DATA_DIR = os.path.join(
43	43	)
44	44
45	45
46		-def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None):
	46	+def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None, project_path=None):
47	47	if not workspace_dir:
48	48	workspace_dir = os.path.join(str(tmpdir), 'workspace{}'.format(suffix))
49		- project_path = os.path.join(datafiles.dirname, datafiles.basename)
	49	+ if not project_path:
	50	+ project_path = os.path.join(datafiles.dirname, datafiles.basename)
	51	+ else:
	52	+ shutil.copytree(os.path.join(datafiles.dirname, datafiles.basename), project_path)
50	53	bin_files_path = os.path.join(project_path, 'files', 'bin-files')
51	54	element_path = os.path.join(project_path, 'elements')
52	55	element_name = 'workspace-test-{}{}.bst'.format(kind, suffix)
...	...	@@ -218,41 +221,42 @@ def test_close(cli, tmpdir, datafiles, kind):
218	221
219	222	@pytest.mark.datafiles(DATA_DIR)
220	223	def test_close_external_after_move_project(cli, tmpdir, datafiles):
221		- tmp_parent = os.path.dirname(str(tmpdir))
222		- workspace_dir = os.path.join(tmp_parent, "workspace")
223		- element_name, project_path, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir)
	224	+ workspace_dir = os.path.join(str(tmpdir), "workspace")
	225	+ project_path = os.path.join(str(tmpdir), 'initial_project')
	226	+ element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False, "", workspace_dir, project_path)
224	227	assert os.path.exists(workspace_dir)
225		- tmp_dir = os.path.join(tmp_parent, 'external_project')
226		- shutil.move(project_path, tmp_dir)
227		- assert os.path.exists(tmp_dir)
	228	+ moved_dir = os.path.join(str(tmpdir), 'external_project')
	229	+ shutil.move(project_path, moved_dir)
	230	+ assert os.path.exists(moved_dir)
228	231
229	232	# Close the workspace
230		- result = cli.run(configure=False, project=tmp_dir, args=[
	233	+ result = cli.run(project=moved_dir, args=[
231	234	'workspace', 'close', '--remove-dir', element_name
232	235	])
233	236	result.assert_success()
234	237
235	238	# Assert the workspace dir has been deleted
236	239	assert not os.path.exists(workspace_dir)
237		- # Move directory back inside tmp directory so it can be recognised
238		- shutil.move(tmp_dir, project_path)
239	240
240	241
241	242	@pytest.mark.datafiles(DATA_DIR)
242	243	def test_close_internal_after_move_project(cli, tmpdir, datafiles):
243		- element_name, project, _ = open_workspace(cli, tmpdir, datafiles, 'git', False)
244		- tmp_dir = os.path.join(os.path.dirname(str(tmpdir)), 'external_project')
245		- shutil.move(str(tmpdir), tmp_dir)
246		- assert os.path.exists(tmp_dir)
	244	+ initial_dir = os.path.join(str(tmpdir), 'initial_project')
	245	+ initial_workspace = os.path.join(initial_dir, 'workspace')
	246	+ element_name, _, _ = open_workspace(cli, tmpdir, datafiles, 'git', False,
	247	+ workspace_dir=initial_workspace, project_path=initial_dir)
	248	+ moved_dir = os.path.join(str(tmpdir), 'internal_project')
	249	+ shutil.move(initial_dir, moved_dir)
	250	+ assert os.path.exists(moved_dir)
247	251
248	252	# Close the workspace
249		- result = cli.run(configure=False, project=tmp_dir, args=[
	253	+ result = cli.run(project=moved_dir, args=[
250	254	'workspace', 'close', '--remove-dir', element_name
251	255	])
252	256	result.assert_success()
253	257
254	258	# Assert the workspace dir has been deleted
255		- workspace = os.path.join(tmp_dir, 'workspace')
	259	+ workspace = os.path.join(moved_dir, 'workspace')
256	260	assert not os.path.exists(workspace)
257	261
258	262

tests/integration/project/elements/sockets/make-builddir-socket.bst

---

	1	+kind: manual
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ build-commands:
	9	+ - |
	10	+ python3 -c '
	11	+ from socket import socket, AF_UNIX, SOCK_STREAM
	12	+ s = socket(AF_UNIX, SOCK_STREAM)
	13	+ s.bind("testsocket")
	14	+ '

tests/integration/project/elements/sockets/make-install-root-socket.bst

---

	1	+kind: manual
	2	+
	3	+depends:
	4	+- filename: base.bst
	5	+ type: build
	6	+
	7	+config:
	8	+ install-commands:
	9	+ - |
	10	+ python3 -c '
	11	+ from os.path import join
	12	+ from sys import argv
	13	+ from socket import socket, AF_UNIX, SOCK_STREAM
	14	+ s = socket(AF_UNIX, SOCK_STREAM)
	15	+ s.bind(join(argv[1], "testsocket"))
	16	+ ' %{install-root}

tests/integration/sockets.py

---

	1	+import os
	2	+import pytest
	3	+
	4	+from buildstream import _yaml
	5	+
	6	+from tests.testutils import cli_integration as cli
	7	+from tests.testutils.integration import assert_contains
	8	+
	9	+
	10	+pytestmark = pytest.mark.integration
	11	+
	12	+DATA_DIR = os.path.join(
	13	+ os.path.dirname(os.path.realpath(__file__)),
	14	+ "project"
	15	+)
	16	+
	17	+
	18	+@pytest.mark.datafiles(DATA_DIR)
	19	+def test_builddir_socket_ignored(cli, tmpdir, datafiles):
	20	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	21	+ element_name = 'sockets/make-builddir-socket.bst'
	22	+
	23	+ result = cli.run(project=project, args=['build', element_name])
	24	+ assert result.exit_code == 0
	25	+
	26	+
	27	+@pytest.mark.datafiles(DATA_DIR)
	28	+def test_install_root_socket_ignored(cli, tmpdir, datafiles):
	29	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	30	+ element_name = 'sockets/make-install-root-socket.bst'
	31	+
	32	+ result = cli.run(project=project, args=['build', element_name])
	33	+ assert result.exit_code == 0

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.