[Notes] [Git][BuildStream/buildstream][edbaunton/executable-remote-source] remote.py: Add support for marking downloaded files executable

[Ed Baunton <gitlab mg gitlab com>]

Title: GitLab

Ed Baunton pushed to branch edbaunton/executable-remote-source at BuildStream / buildstream

Commits:

• 3c2d208f
  by Ed Baunton at 2018-07-27T13:04:00Z

  remote.py: Add support for marking downloaded files executable
  
  Add an optional flag to make files executable after having downloaded them.
  

3 changed files:

• buildstream/plugins/sources/remote.py
• tests/sources/remote.py
• + tests/sources/remote/unique-keys/target-custom-executable.bst

Changes:

buildstream/plugins/sources/remote.py

---

...	...	@@ -35,6 +35,10 @@ remote - stage files from remote urls
35	35	# If not specified, the basename of the url will be used.
36	36	# filename: customfilename
37	37
	38	+ # Optionally specify whether the downloaded file should be
	39	+ # marked executable.
	40	+ # executable: true
	41	+
38	42	# Specify the url. Using an alias defined in your project
39	43	# configuration is encouraged. 'bst track' will update the
40	44	# sha256sum in 'ref' to the downloaded file's sha256sum.
...	...	@@ -43,16 +47,18 @@ remote - stage files from remote urls
43	47	# Specify the ref. It's a sha256sum of the file you download.
44	48	ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
45	49
	50	+
	51	+
46	52	.. note::
47	53
48	54	The ``remote`` plugin is available since :ref:`format version 10 <project_format_version>`
49	55
50	56	"""
51	57	import os
	58	+import stat
52	59	from buildstream import SourceError, utils
53	60	from ._downloadablefilesource import DownloadableFileSource
54	61
55		-
56	62	class RemoteSource(DownloadableFileSource):
57	63	# pylint: disable=attribute-defined-outside-init
58	64
...	...	@@ -60,22 +66,27 @@ class RemoteSource(DownloadableFileSource):
60	66	super().configure(node)
61	67
62	68	self.filename = self.node_get_member(node, str, 'filename', os.path.basename(self.url))
	69	+ self.executable = self.node_get_member(node, bool, 'executable', False)
63	70
64	71	if os.sep in self.filename:
65	72	raise SourceError('{}: filename parameter cannot contain directories'.format(self),
66	73	reason="filename-contains-directory")
67		- self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename'])
	74	+ self.node_validate(node, DownloadableFileSource.COMMON_CONFIG_KEYS + ['filename', 'executable'])
68	75
69	76	def get_unique_key(self):
70		- return super().get_unique_key() + [self.filename]
	77	+ return super().get_unique_key() + [self.filename, self.executable]
71	78
72	79	def stage(self, directory):
73	80	# Same as in local plugin, don't use hardlinks to stage sources, they
74	81	# are not write protected in the sandbox.
75	82	dest = os.path.join(directory, self.filename)
76	83	with self.timed_activity("Staging remote file to {}".format(dest)):
	84	+
77	85	utils.safe_copy(self._get_mirror_file(), dest)
78	86
	87	+ if self.executable:
	88	+ st = os.stat(dest)
	89	+ os.chmod(dest, st.st_mode | stat.S_IEXEC)
79	90
80	91	def setup():
81	92	return RemoteSource

tests/sources/remote.py

---

1	1	import os
	2	+import stat
2	3	import pytest
3	4
4	5	from buildstream._exceptions import ErrorDomain
...	...	@@ -82,7 +83,10 @@ def test_simple_file_build(cli, tmpdir, datafiles):
82	83	result.assert_success()
83	84	# Note that the url of the file in target.bst is actually /dir/file
84	85	# but this tests confirms we take the basename
85		- assert(os.path.exists(os.path.join(checkoutdir, 'file')))
	86	+ checkout_file = os.path.join(checkoutdir, 'file')
	87	+ assert(os.path.exists(checkout_file))
	88	+
	89	+ assert(not (os.stat(checkout_file).st_mode & stat.S_IEXEC))
86	90
87	91
88	92	@pytest.mark.datafiles(os.path.join(DATA_DIR, 'single-file-custom-name'))
...	...	@@ -119,6 +123,7 @@ def test_unique_key(cli, tmpdir, datafiles):
119	123	generate_project(project, tmpdir)
120	124	assert cli.get_element_state(project, 'target.bst') == "fetch needed"
121	125	assert cli.get_element_state(project, 'target-custom.bst') == "fetch needed"
	126	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
122	127	# Try to fetch it
123	128	result = cli.run(project=project, args=[
124	129	'fetch', 'target.bst'
...	...	@@ -127,7 +132,29 @@ def test_unique_key(cli, tmpdir, datafiles):
127	132	# We should download the file only once
128	133	assert cli.get_element_state(project, 'target.bst') == 'buildable'
129	134	assert cli.get_element_state(project, 'target-custom.bst') == 'buildable'
	135	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == 'buildable'
130	136
131	137	# But the cache key is different because the 'filename' is different.
132	138	assert cli.get_element_key(project, 'target.bst') != \
133		- cli.get_element_key(project, 'target-custom.bst')
	139	+ cli.get_element_key(project, 'target-custom.bst') != \
	140	+ cli.get_element_key(project, 'target-custom-executable.bst')
	141	+
	142	+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'unique-keys'))
	143	+def test_executable(cli, tmpdir, datafiles):
	144	+ '''This test confirms that the 'ecxecutable' parameter is honoured.
	145	+ '''
	146	+ project = os.path.join(datafiles.dirname, datafiles.basename)
	147	+ generate_project(project, tmpdir)
	148	+ checkoutdir = os.path.join(str(tmpdir), "checkout")
	149	+ assert cli.get_element_state(project, 'target-custom-executable.bst') == "fetch needed"
	150	+ # Try to fetch it
	151	+ result = cli.run(project=project, args=[
	152	+ 'build', 'target-custom-executable.bst'
	153	+ ])
	154	+
	155	+ result = cli.run(project=project, args=[
	156	+ 'checkout', 'target-custom-executable.bst', checkoutdir
	157	+ ])
	158	+
	159	+ assert (os.stat(
	160	+ os.path.join(checkoutdir,'some-custom-file')).st_mode & stat.S_IEXEC)

tests/sources/remote/unique-keys/target-custom-executable.bst

---

	1	+kind: import
	2	+description: test
	3	+sources:
	4	+- kind: remote
	5	+ url: tmpdir:/dir/file
	6	+ ref: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
	7	+ filename: some-custom-file
	8	+ executable: true

—
View it on GitLab.
You're receiving this email because of your account on gitlab.com. If you'd like to receive fewer emails, you can adjust your notification settings.